{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# HTML解析入门及准备URL生成连续技\n",
    "\n",
    "*  本周主要内容：批量抓取页面基础及技巧\n",
    "*  上周主要内容：HTML解析（parse HTML）及准备URL生成连续技\n",
    "*  20春_Web数据挖掘_week04\n",
    "*  电子讲义设计者：廖汉腾, 许智超\n",
    "<br/>\n",
    "<br/>\n",
    "\n",
    "-----\n",
    "## 复习\n",
    "\n",
    "复习：上周内容，实践\n",
    "\n",
    "* 翻页：参数字典的拆解\n",
    "* 翻页：系统性迭代\n",
    "* 翻页：数据备份与整合\n",
    "\n",
    "-----\n",
    "## 本周内容及学习目标\n",
    "\n",
    "搜狗微信：搜索关键字获取微信公众号相关文章\n",
    "  \n",
    "### 目标\n",
    "1. 使用 requests-html 爬取并存取网页文字档，查找[requests-html 中文文档](https://cncert.github.io/requests-html-doc-cn/#/)\n",
    "2. 熟悉 [xpath 语法](https://www.w3cschool.cn/xpath/xpath-syntax.html)丶[xpath 节点](https://www.w3cschool.cn/xpath/xpath-nodes.html)\n",
    "3. 使用 [xpath cheatsheet](https://devhints.io/xpath)\n",
    "  * 在 Chrome Inspector 使用\n",
    "  * 在 requests-html (Python) 使用\n",
    "4. 简易使用 [pd.DataFrame](https://www.pypandas.cn/doc/getting_started/dsintro.html#dataframe)\n",
    "5. 参数字典的拆解与迭代\n",
    "6. 翻页数据备份与整合"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>\n",
       "/* 本电子讲义使用之CSS */\n",
       "div.code_cell {\n",
       "    background-color: #e5f1fe;\n",
       "}\n",
       "div.cell.selected {\n",
       "    background-color: #effee2;\n",
       "    font-size: 2rem;\n",
       "    line-height: 2.4rem;\n",
       "}\n",
       "div.cell.selected .rendered_html table {\n",
       "    font-size: 2rem !important;\n",
       "    line-height: 2.4rem !important;\n",
       "}\n",
       ".rendered_html pre code {\n",
       "    background-color: #C4E4ff;   \n",
       "    padding: 2px 25px;\n",
       "}\n",
       ".rendered_html pre {\n",
       "    background-color: #99c9ff;\n",
       "}\n",
       "div.code_cell .CodeMirror {\n",
       "    font-size: 2rem !important;\n",
       "    line-height: 2.4rem !important;\n",
       "}\n",
       ".rendered_html img, .rendered_html svg {\n",
       "    max-width: 60%;\n",
       "    height: auto;\n",
       "    float: right;\n",
       "}\n",
       "\n",
       ".rendered_html img[src*=\"#full\"], .rendered_html svg[src*=\"#full\"] {\n",
       "    max-width: 100%;\n",
       "    height: auto;\n",
       "    float: none;\n",
       "}\n",
       "\n",
       ".rendered_html img[src*=\"#thumbnail\"], .rendered_html svg[src*=\"#thumbnail\"] {\n",
       "    max-width: 15%;\n",
       "    height: auto;\n",
       "}\n",
       "\n",
       "/* Gradient transparent - color - transparent */\n",
       "hr {\n",
       "    border: 0;\n",
       "    border-bottom: 1px dashed #ccc;\n",
       "}\n",
       ".emoticon{\n",
       "    font-size: 5rem;\n",
       "    line-height: 4.4rem;\n",
       "    text-align: center;\n",
       "    vertical-align: middle;\n",
       "}\n",
       ".bg-split_apply_comine {\n",
       "    width: 500px;     \n",
       "    height: 300px;\n",
       "    background: url('02_split-apply-comine_500x300.png') -10px -10px;\n",
       "    float: right;\n",
       "}\n",
       ".bg-comine {\n",
       "    width: 175px;\n",
       "    height: 150px;\n",
       "    background: url('02_split-apply-comine_500x300.png') -280px -80px;\n",
       "    float: right;\n",
       "}\n",
       ".bg-apply {\n",
       "    width: 155px;\n",
       "    height: 225px;\n",
       "    background: url('02_split-apply-comine_500x300.png') -160px -30px;\n",
       "    float: right;\n",
       "}\n",
       ".bg-split {\n",
       "    width: 205px;\n",
       "    height: 225px;\n",
       "    background: url('02_split-apply-comine_500x300.png') -10px -30px;\n",
       "    float: right;\n",
       "}\n",
       ".break {\n",
       "                   page-break-after: right; \n",
       "                   width:700px;\n",
       "                   clear:both;\n",
       "}\n",
       "</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%html\n",
    "<style>\n",
    "/* 本电子讲义使用之CSS */\n",
    "div.code_cell {\n",
    "    background-color: #e5f1fe;\n",
    "}\n",
    "div.cell.selected {\n",
    "    background-color: #effee2;\n",
    "    font-size: 2rem;\n",
    "    line-height: 2.4rem;\n",
    "}\n",
    "div.cell.selected .rendered_html table {\n",
    "    font-size: 2rem !important;\n",
    "    line-height: 2.4rem !important;\n",
    "}\n",
    ".rendered_html pre code {\n",
    "    background-color: #C4E4ff;   \n",
    "    padding: 2px 25px;\n",
    "}\n",
    ".rendered_html pre {\n",
    "    background-color: #99c9ff;\n",
    "}\n",
    "div.code_cell .CodeMirror {\n",
    "    font-size: 2rem !important;\n",
    "    line-height: 2.4rem !important;\n",
    "}\n",
    ".rendered_html img, .rendered_html svg {\n",
    "    max-width: 60%;\n",
    "    height: auto;\n",
    "    float: right;\n",
    "}\n",
    "\n",
    ".rendered_html img[src*=\"#full\"], .rendered_html svg[src*=\"#full\"] {\n",
    "    max-width: 100%;\n",
    "    height: auto;\n",
    "    float: none;\n",
    "}\n",
    "\n",
    ".rendered_html img[src*=\"#thumbnail\"], .rendered_html svg[src*=\"#thumbnail\"] {\n",
    "    max-width: 15%;\n",
    "    height: auto;\n",
    "}\n",
    "\n",
    "/* Gradient transparent - color - transparent */\n",
    "hr {\n",
    "    border: 0;\n",
    "    border-bottom: 1px dashed #ccc;\n",
    "}\n",
    ".emoticon{\n",
    "    font-size: 5rem;\n",
    "    line-height: 4.4rem;\n",
    "    text-align: center;\n",
    "    vertical-align: middle;\n",
    "}\n",
    ".bg-split_apply_comine {\n",
    "    width: 500px;     \n",
    "    height: 300px;\n",
    "    background: url('02_split-apply-comine_500x300.png') -10px -10px;\n",
    "    float: right;\n",
    "}\n",
    ".bg-comine {\n",
    "    width: 175px;\n",
    "    height: 150px;\n",
    "    background: url('02_split-apply-comine_500x300.png') -280px -80px;\n",
    "    float: right;\n",
    "}\n",
    ".bg-apply {\n",
    "    width: 155px;\n",
    "    height: 225px;\n",
    "    background: url('02_split-apply-comine_500x300.png') -160px -30px;\n",
    "    float: right;\n",
    "}\n",
    ".bg-split {\n",
    "    width: 205px;\n",
    "    height: 225px;\n",
    "    background: url('02_split-apply-comine_500x300.png') -10px -30px;\n",
    "    float: right;\n",
    "}\n",
    ".break {\n",
    "                   page-break-after: right; \n",
    "                   width:700px;\n",
    "                   clear:both;\n",
    "}\n",
    "</style>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 330,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 基本模块\n",
    "import pandas as pd\n",
    "from requests_html import HTMLSession\n",
    "from pandas.api.types import is_numeric_dtype"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 搜狗微信：文章搜索(模仿liepin)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 533,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020年UI/UX九大流行趋势</td>\n",
       "      <td></td>\n",
       "      <td>互联网架构师</td>\n",
       "      <td>1585701057</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-01 00:30:57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>#不E样的心机体验#焕新UX设计体验如何?话筒给他们!</td>\n",
       "      <td>今天,我们一起来看看花粉朋友们对于EMUI10“美,不止初见”的UX设计有哪些使用感受吧!焕...</td>\n",
       "      <td>华为EMUI</td>\n",
       "      <td>1582106885</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-02-19 10:08:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>手机增强现实应用程序的UX设计规范</td>\n",
       "      <td>为了确保未来MAR系统被认可和成功,也为了填补MAR研究的空白,确定适用于MAR应用的当下U...</td>\n",
       "      <td>MicroUX</td>\n",
       "      <td>1583817775</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-03-10 05:22:55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>全新雷克萨斯纯电动UX 300e 开启预热</td>\n",
       "      <td>全新雷克萨斯纯电动UX 300e于广州车展全球首发,经过几个月的等待,即日起全面开启预热.欢...</td>\n",
       "      <td>临沂和凌雷克萨斯4S店</td>\n",
       "      <td>1586830340</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-14 02:12:20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2020年UI/UX九大流行趋势</td>\n",
       "      <td></td>\n",
       "      <td>前端达人</td>\n",
       "      <td>1584765310</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-03-21 04:35:10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>选口碑豪华SUV,XT4、冒险家、UX的优势在哪里?</td>\n",
       "      <td>UX售价28.30万元起,假如它能再多多提供终端优惠,我相信它也能和加价才能提车的兄弟ES一...</td>\n",
       "      <td>车闻社</td>\n",
       "      <td>1585218357</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-03-26 10:25:57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>【视频】2020 UX设计趋势</td>\n",
       "      <td>“用户体验”是过去十年的流行语之一,在过去的几年中,UX已经从设计世界的另一个口号变成了世界...</td>\n",
       "      <td>设计帝国网站</td>\n",
       "      <td>1585702828</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-01 01:00:28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>UX.LAB实验室-防疫设计大赛开始了!</td>\n",
       "      <td>UX.LAB实验室防疫设计大赛开始了!(防疫设计主题作品示例,素材来源自花瓣网)01活动目的...</td>\n",
       "      <td>用户体验实验室</td>\n",
       "      <td>1585717688</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-01 05:08:08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2020年9大 UX设计趋势</td>\n",
       "      <td></td>\n",
       "      <td>美啊教育</td>\n",
       "      <td>1585648837</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-03-31 10:00:37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>雷克萨斯加速电气化战略 首款纯电动车UX 300e年内进入中国及欧洲市场 | 经观汽车</td>\n",
       "      <td>王海宣受新冠肺炎疫情影响,日内瓦车展被迫取消,而雷克萨斯原本在日内瓦车展亮相的旗下首款纯电动...</td>\n",
       "      <td>经观汽车</td>\n",
       "      <td>1583476788</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-03-06 06:39:48</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            标题  \\\n",
       "0                             2020年UI/UX九大流行趋势   \n",
       "1                  #不E样的心机体验#焕新UX设计体验如何?话筒给他们!   \n",
       "2                            手机增强现实应用程序的UX设计规范   \n",
       "3                        全新雷克萨斯纯电动UX 300e 开启预热   \n",
       "4                             2020年UI/UX九大流行趋势   \n",
       "5                   选口碑豪华SUV,XT4、冒险家、UX的优势在哪里?   \n",
       "6                              【视频】2020 UX设计趋势   \n",
       "7                         UX.LAB实验室-防疫设计大赛开始了!   \n",
       "8                               2020年9大 UX设计趋势   \n",
       "9  雷克萨斯加速电气化战略 首款纯电动车UX 300e年内进入中国及欧洲市场 | 经观汽车   \n",
       "\n",
       "                                                  摘要       source          时间  \\\n",
       "0                                                          互联网架构师  1585701057   \n",
       "1  今天,我们一起来看看花粉朋友们对于EMUI10“美,不止初见”的UX设计有哪些使用感受吧!焕...       华为EMUI  1582106885   \n",
       "2  为了确保未来MAR系统被认可和成功,也为了填补MAR研究的空白,确定适用于MAR应用的当下U...      MicroUX  1583817775   \n",
       "3  全新雷克萨斯纯电动UX 300e于广州车展全球首发,经过几个月的等待,即日起全面开启预热.欢...  临沂和凌雷克萨斯4S店  1586830340   \n",
       "4                                                            前端达人  1584765310   \n",
       "5  UX售价28.30万元起,假如它能再多多提供终端优惠,我相信它也能和加价才能提车的兄弟ES一...          车闻社  1585218357   \n",
       "6  “用户体验”是过去十年的流行语之一,在过去的几年中,UX已经从设计世界的另一个口号变成了世界...       设计帝国网站  1585702828   \n",
       "7  UX.LAB实验室防疫设计大赛开始了!(防疫设计主题作品示例,素材来源自花瓣网)01活动目的...      用户体验实验室  1585717688   \n",
       "8                                                            美啊教育  1585648837   \n",
       "9  王海宣受新冠肺炎疫情影响,日内瓦车展被迫取消,而雷克萨斯原本在日内瓦车展亮相的旗下首款纯电动...         经观汽车  1583476788   \n",
       "\n",
       "                                               公众号链接  \\\n",
       "0  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "1  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "2  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "3  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "4  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "5  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "6  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "7  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "8  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "9  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "\n",
       "                                                文章链接                格式化时间  \n",
       "0  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-01 00:30:57  \n",
       "1  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-02-19 10:08:05  \n",
       "2  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-03-10 05:22:55  \n",
       "3  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-14 02:12:20  \n",
       "4  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-03-21 04:35:10  \n",
       "5  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-03-26 10:25:57  \n",
       "6  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-01 01:00:28  \n",
       "7  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-01 05:08:08  \n",
       "8  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-03-31 10:00:37  \n",
       "9  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-03-06 06:39:48  "
      ]
     },
     "execution_count": 533,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# A-1 query 为关键字选取 以python为例\n",
    "\n",
    "url = \"https://weixin.sogou.com/weixin\"\n",
    "session = HTMLSession()\n",
    "payload = {\n",
    "    \"query\":\"UX\",\n",
    "    \"type\":\"2\", # 2 文章、1 公众号\n",
    "    \"page\":10  \n",
    "}\n",
    "\n",
    "r = session.get( url, params = payload, )\n",
    "\n",
    "\n",
    "主要元素 =  r.html.xpath( '//ul[@class=\"news-list\"]/li')\n",
    "\n",
    "dict_xpaths = {\n",
    "    'text':{\n",
    "        'source':'//div[contains(@class,\"txt-box\")]/div[@class=\"s-p\"]/a/text()',\n",
    "        '时间':'//div[contains(@class,\"txt-box\")]/div[@class=\"s-p\"]/@t',       \n",
    "    },\n",
    "    'text_content':{\n",
    "        '标题':'//div[contains(@class,\"txt-box\")]/h3/a',\n",
    "        '摘要':'//div[contains(@class,\"txt-box\")]/p'\n",
    "    },\n",
    "    'href':{\n",
    "        \"公众号链接\":'//div[contains(@class,\"txt-box\")]/div[@class=\"s-p\"]/a/@href',\n",
    "        \"文章链接\":'//div[contains(@class,\"txt-box\")]/h3/a/@href',\n",
    "    }\n",
    "}\n",
    "\n",
    "\n",
    "def get_e_text_content(_xpath_):\n",
    "    # 高级列表推导\n",
    "    暂存结果 = [e.xpath(_xpath_)[0].lxml.text_content() for e in 主要元素]\n",
    "\n",
    "    return(暂存结果)\n",
    "\n",
    "def get_e_text(_xpath_):\n",
    "    暂存结果 = [\"\".join([x.strip() if type(x) is str else x.text.strip() for x in e.xpath(_xpath_)]) for e in 主要元素]\n",
    "    return(暂存结果)\n",
    "\n",
    "def get_e_href(_xpath_):\n",
    "    # 高级列表推导\n",
    "    暂存结果 = [\"https://weixin.sogou.com\"+item\\\n",
    "            for item in e.xpath(_xpath_)\\\n",
    "            for e in 主要元素]\n",
    "    return(暂存结果)\n",
    "\n",
    "数据字典 = dict()\n",
    "# content\n",
    "数据字典 = {k:get_e_text_content(v) for k,v in dict_xpaths['text_content'].items()}\n",
    "# text\n",
    "数据字典.update({k:get_e_text(v) for k,v in dict_xpaths['text'].items()})\n",
    "# href\n",
    "数据字典.update({k:get_e_href(v) for k,v in dict_xpaths['href'].items()})\n",
    "\n",
    "数据 = pd.DataFrame(数据字典)\n",
    "数据 = 数据.assign(\n",
    "    格式化时间 = lambda x: pd.to_datetime(x[\"时间\"], unit='s').dt.strftime('%Y-%m-%d %H:%M:%S')\n",
    ")\n",
    "数据\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 531,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open (\"weixin_UX.html\", encoding = \"utf8\", mode = \"w\") as fp:\n",
    "    fp.write(r.html.html)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## xpath的具体操作"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 493,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['CSDN',\n",
       " 'CSDN',\n",
       " 'Python数据之道',\n",
       " '大学生必备网',\n",
       " '寻梦暖屋',\n",
       " 'Python程序员',\n",
       " 'InfoQ',\n",
       " 'Python那些事',\n",
       " 'Python小屋',\n",
       " 'Python程序员']"
      ]
     },
     "execution_count": 493,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 来源的xpath解析结果\n",
    "r.html.xpath('//div[contains(@class,\"txt-box\")]/div[@class=\"s-p\"]/a/text()')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 495,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[' 十大装 B 语法',\n",
       " '势头强劲的 ',\n",
       " ' PK 强大的 C++,究竟谁更胜一筹?',\n",
       " '很冷门,用起来却非常实用',\n",
       " '库',\n",
       " '9.9元',\n",
       " '小课体验课,限100人!',\n",
       " '今日学习-',\n",
       " '集合(1)',\n",
       " '找对象那么难?',\n",
       " ' s3cmd 帮你解决',\n",
       " '为什么 ',\n",
       " ' 不是未来的编程语言?',\n",
       " '7 款 ',\n",
       " ' 开源框架的优劣总结',\n",
       " '绘制傅里叶变换、反变换与带通滤波图像',\n",
       " '用',\n",
       " '模拟新冠肺炎的传播']"
      ]
     },
     "execution_count": 495,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 标题？1、em text()? + a/text()?\n",
    "# 某一条<li>信息,再进行上一步 if else\n",
    "r.html.xpath('//div[contains(@class,\"txt-box\")]/h3/a/text()')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 500,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Python 十大装 B 语法'"
      ]
     },
     "execution_count": 500,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.html.xpath('//div[contains(@class,\"txt-box\")]/h3/a')[0].lxml.text_content()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 502,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'作者 | 许向武责编 | 郭芮出品 | CSDN 博客Python 是一种代表简单思想的语言,其语法相对简单,很容易上手.不过,如果就此小视 ...'"
      ]
     },
     "execution_count": 502,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.html.xpath('//div[contains(@class,\"txt-box\")]/p')[0].lxml.text_content()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 503,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPq5hmM2JyHRhS2xrPG4asQ7AwvDqyjOWdzeLxXplXAcuNuVuNPiFqscsKWDWntqkEi14zy27r5NbdFls1_OnGoFix4ZDKkBJOx-2Ls84MFeMX_AicIp_zOvFbsuFhbVQ2Rthlp8MQWpUw2Ae8Afa8duOO00efWrWmm&type=2&query=python&token=18C534345BEFB54D9F9A3C97CC9C9383A01FE1A55E9525FE',\n",
       " '/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPq5hmM2JyHRhS2xrPG4asQ7AwvDqyjOWdzeLxXplXAcuNuVuNPiFqscsKWDWntqkEi14zy27r5NbdFls1_OnGoFix4ZDKkBJOx-2Ls84MFeMX_AicIp_zOvFbsuFhbVQ2Rthlp8MQWpUw2Ae8Afa8duOO00efWrWmm&type=2&query=python&token=18C534345BEFB54D9F9A3C97CC9C9383A01FE1A55E9525FE',\n",
       " '/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPq5hmM2JyHRhS2xrPG4asQ7AwvDqyjOWdz0w9ljRZpbI9M_GboL8uxYw0aK1zsA837eBDt0gCrk0YQB5oodUcHb3fERD_b9RnkhAyc1Bu02V_mR1_SalCO2XhqiE_LS1nxTzOB9g_EDYCxvSF_admVdGRTpLLTNgHY&type=2&query=python&token=18C534345BEFB54D9F9A3C97CC9C9383A01FE1A55E9525FE',\n",
       " '/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPq5hmM2JyHRhS2xrPG4asQ7AwvDqyjOWdzKQ7XL2LD0OiR8KGlYIFUDnWemq3ttMblIeHxnG_KaRP50wOnNaswJK9c3Yf4JYnM2cc-y5RzE8oq9WbvnRurCwH7kT84ATISW1D_ecK7DYw-1mRcxTyg_uO00efWrWmm&type=2&query=python&token=18C534345BEFB54D9F9A3C97CC9C9383A01FE1A55E9525FE',\n",
       " '/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPq5hmM2JyHRhS2xrPG4asQ7AwvDqyjOWdzdwnw8aAVpWXSHE52syYftbUNhupjB3o2CHDUU9SosASKpozjjFFEvkQ-gCwGsfauuxArGlAC1VhwnE1snCzL46f0aU_2kfhoE9chpjNwQbY0akJHUyMz02RTpLLTNgHY&type=2&query=python&token=18C534345BEFB54D9F9A3C97CC9C9383A01FE1A55E9525FE',\n",
       " '/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPq5hmM2JyHRhS2xrPG4asQ7AwvDqyjOWdzAR7ilSfouClB09PY7DYa77OO--NSom_qLZ2wQ0yILNSWLuqPYtzoZjquzc4itwQaZ_S8Z1ArJ8TCVXxsHHaSybf_ncM8D9b3PqnQ6nt3NlUngDxg8LCCz2RTpLLTNgHY&type=2&query=python&token=18C534345BEFB54D9F9A3C97CC9C9383A01FE1A55E9525FE',\n",
       " '/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPq5hmM2JyHRhS2xrPG4asQ7AwvDqyjOWdzGHeCqZF0mi7y0NFgb-8tUDvrsDGtKAUkPQrKMnKgWmqFS8o3O0_HNK83DDsdDn2Bs09EbEpAmddk1D1n58qlsMCyKnM2i5GAZzYChakz2F_wcijPZ-S2pG4OZCnxixtX&type=2&query=python&token=18C534345BEFB54D9F9A3C97CC9C9383A01FE1A55E9525FE',\n",
       " '/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPq5hmM2JyHRhS2xrPG4asQ7AwvDqyjOWdzzkX9GjFmCFUx9xnDxF3lNmbaZkaVTpQbuGKAjzePCZSDONChY1OIJW_icXmAeMz4F8ekQ6AZ_67Btn091J4VvU-4CM2xbC7RdXhsGu51TmBNFOmj4-9mDuO00efWrWmm&type=2&query=python&token=18C534345BEFB54D9F9A3C97CC9C9383A01FE1A55E9525FE',\n",
       " '/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPq5hmM2JyHRhS2xrPG4asQ7AwvDqyjOWdzYd0wMP9_-EPDat3VZO_kBre35K4rbvRoFFMPq2isgQoLGzLVzCJ68VCEervZitLcOGWngIW7BvG4m91IcGiinh_kRbpcLN0cNfnrqR0W0ZuwTpto1p08f2RTpLLTNgHY&type=2&query=python&token=18C534345BEFB54D9F9A3C97CC9C9383A01FE1A55E9525FE',\n",
       " '/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPq5hmM2JyHRhS2xrPG4asQ7AwvDqyjOWdzAR7ilSfouClB09PY7DYa77OO--NSom_qLZ2wQ0yILNSWLuqPYtzoZjquzc4itwQaZ_S8Z1ArJ8TCVXxsHHaSybf_ncM8D9b3PqnQ6nt3NlUngDxg8LCCz2RTpLLTNgHY&type=2&query=python&token=18C534345BEFB54D9F9A3C97CC9C9383A01FE1A55E9525FE']"
      ]
     },
     "execution_count": 503,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 公众号链接\n",
    "'https://weixin.sogou.com/weixin'+r.html.xpath('//div[contains(@class,\"txt-box\")]/div[@class=\"s-p\"]/a/@href')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 公众号文章链接"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 时间"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 图片"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 489,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 Safari/603.3.8', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive', 'Cookie': 'IPLOC=CN4419; SNUID=AEDFDD61181CBA2A31789BFB19535630; SUID=B6C6C5784A42910A000000005E94CF61; ABTEST=8|1586810721|v1'}"
      ]
     },
     "execution_count": 489,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.request.headers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 486,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A-2 下载页面\n",
    "with open (\"weixin_用户研究.html\", encoding = \"utf8\", mode = \"w\") as fp:\n",
    "    fp.write(r.html.html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 444,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1.库引用与import函数2.库引用:扩充Python程序功能的方式3.使用import保留字,采用. ()的风格4.import    . (函数参数)5.使用保...'"
      ]
     },
     "execution_count": 444,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# lxml.text_content()\n",
    "r.html.xpath('//div[contains(@class,\"txt-box\")]/h3/a/em/text()')\n",
    "r.html.xpath('//div[contains(@class,\"txt-box\")]/h3/a/text()')\n",
    "r.html.xpath('//div[contains(@class,\"txt-box\")]/p')[0].lxml.text_content()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 446,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqDMsPE5ZFEm9svg4Skk4qHgwvDqyjOWdzqJJllMV4sgV_iqjAviJMH-nun9V5cWucQzQ0H3lPk1SaWMC6z7zW3ZuD2Fblyv0UPqDp0JAtd4YsPXStQJx02ZcC0X5jBZq0XqaPxPMr9WBrN6ixXvK44WRTpLLTNgHY&type=2&query=python&token=161181389B20209DE6E046D7D1CD6454E666492D5E94BF4F',\n",
       " '/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqDMsPE5ZFEm9svg4Skk4qHgwvDqyjOWdzM1N3sg6-MGMs4Ovzv0sOdV6iNSk94nbViiSSJhj5RDOB07es2XIcPyyXi_McJDQhaXt1_F979mkpTPofR_EpXMdI9FUlMBCcnD1ESuzbUbKvPER_fgAHB-O00efWrWmm&type=2&query=python&token=161181389B20209DE6E046D7D1CD6454E666492D5E94BF4F',\n",
       " '/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqDMsPE5ZFEm9svg4Skk4qHgwvDqyjOWdzM1N3sg6-MGMs4Ovzv0sOdV6iNSk94nbViiSSJhj5RDOB07es2XIcPyyXi_McJDQhaXt1_F979mkpTPofR_EpXMdI9FUlMBCcnD1ESuzbUbKvPER_fgAHB-O00efWrWmm&type=2&query=python&token=161181389B20209DE6E046D7D1CD6454E666492D5E94BF4F',\n",
       " '/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqDMsPE5ZFEm9svg4Skk4qHgwvDqyjOWdzru_EY6bPTmC5nQuwwtv_jt7qRm7G20nMNdw901tEjyHm8BtiwifZh0Gf2bd08kgxkAWXVPzJF94UmmUs9iUFy5A2kY6ZTI_afGe609JtqeMfHDE8YJvFBWRTpLLTNgHY&type=2&query=python&token=161181389B20209DE6E046D7D1CD6454E666492D5E94BF4F',\n",
       " '/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqDMsPE5ZFEm9svg4Skk4qHgwvDqyjOWdzeLxXplXAcuNuVuNPiFqscsKWDWntqkEi14zy27r5NbdFls1_OnGoFix4ZDKkBJOx-2Ls84MFeMX_AicIp_zOvPRAm5gSM4PGGt0NymnQzOvXQPocgszCzqUiiWOv1GPM&type=2&query=python&token=161181389B20209DE6E046D7D1CD6454E666492D5E94BF4F',\n",
       " '/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqDMsPE5ZFEm9svg4Skk4qHgwvDqyjOWdz70wgq3R0pDiWQXgRHTPTr93hpTObNtfaH4NenXWFWBR77T5pZZyp5x_STn8LzPVZkBcYHRFnGiOxElVC9iimFW87aaSA3mXkIAcziDEzKPTdZKc7qCHGg2RTpLLTNgHY&type=2&query=python&token=161181389B20209DE6E046D7D1CD6454E666492D5E94BF4F',\n",
       " '/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqDMsPE5ZFEm9svg4Skk4qHgwvDqyjOWdzeLxXplXAcuNuVuNPiFqscsKWDWntqkEi14zy27r5NbdFls1_OnGoFix4ZDKkBJOx-2Ls84MFeMX_AicIp_zOvPRAm5gSM4PGGt0NymnQzOvXQPocgszCzqUiiWOv1GPM&type=2&query=python&token=161181389B20209DE6E046D7D1CD6454E666492D5E94BF4F',\n",
       " '/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqDMsPE5ZFEm9svg4Skk4qHgwvDqyjOWdzMLX48LkMcFa2V21aBrFhB_2d6r5zx5EqpEdXs3XN9R2NagkYJgv8hd_JtxHZngmb1WkFNz1bLhwOBai_1OwhCGeMJpTIWq_KCrGW7gzpQFZEyudtBcWpsG4OZCnxixtX&type=2&query=python&token=161181389B20209DE6E046D7D1CD6454E666492D5E94BF4F',\n",
       " '/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqDMsPE5ZFEm9svg4Skk4qHgwvDqyjOWdzTl_qLrRJ2EwDKg_T1RTkVaZN8Yz4HLToNg3d6NLR8lsU2ebEQ6S3Aj5GHpH6hRCTptTOpDsjieXZ_Bg3F-9_x0cXMxIZHW9jnKmhtHVwkwjhbG48ZhrZC2RTpLLTNgHY&type=2&query=python&token=161181389B20209DE6E046D7D1CD6454E666492D5E94BF4F',\n",
       " '/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqDMsPE5ZFEm9svg4Skk4qHgwvDqyjOWdzzkX9GjFmCFUx9xnDxF3lNmbaZkaVTpQbuGKAjzePCZSDONChY1OIJW_icXmAeMz4F8ekQ6AZ_67Btn091J4VvTm2ijK8fMjxideUgp7rFUAOxZ3AQe_RN24OZCnxixtX&type=2&query=python&token=161181389B20209DE6E046D7D1CD6454E666492D5E94BF4F']"
      ]
     },
     "execution_count": 446,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.html.xpath('//div[contains(@class,\"txt-box\")]/h3/a/@href')\n",
    "r.html.xpath('//div[contains(@class,\"txt-box\")]/div[@class=\"s-p\"]/a/@href')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 409,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 Safari/603.3.8', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive', 'Cookie': 'ABTEST=0|1586803293|v1'}"
      ]
     },
     "execution_count": 409,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.request.headers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 下载页面和输出excel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 405,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A-2 下载页面\n",
    "with open (\"weixin_python.html\", encoding = \"utf8\", mode = \"w\") as fp:\n",
    "    fp.write(r.html.html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 337,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/xuzhichao/anaconda3/lib/python3.7/site-packages/xlsxwriter/worksheet.py:910: UserWarning: Ignoring URL 'http://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqR9qmSowG7r2gBNPyrh3FVgwvDqyjOWdzeOCKnRJAxkUdZw5uyu3Hynw3YU8OWoUNYcaOf58YCgqXH6IVD2Cttu-1hfDLPAhT5AG8baJOzSK67Mho2fdeLHDctAPZVNzluOsufU76d3ec5RqUeByHO-O00efWrWmm&type=2&query=python&token=210ED24AF7431EDE0C0EA8225BA57DCD0C67D9EF5E948ED8' with link or location/anchor > 255 characters since it exceeds Excel's limit for URLS\n",
      "  force_unicode(url))\n",
      "/Users/xuzhichao/anaconda3/lib/python3.7/site-packages/xlsxwriter/worksheet.py:910: UserWarning: Ignoring URL 'http://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6cwJThYulHEtVjXrGTiVgS9dZesVPC425bopQRjCb2BlpQ4aYX0iWk1qXa8Fplpd9Hhup-WCiA6jKWLk-8yqbp2an3quTv2SP3gDmUfMBWk4hAl2leQHjMIc7a1X13FuuMHylJLJweSgqKYEhdkYntOId8_j3Lu1YmkUd6oG33m7wXpIptlAt7U6T7RdGxQgLBllaLKbmvInP_dLDClI7biNRhieJZTrlVKy4Eeemcj_f3q8uRDTfkw..&type=2&query=python&token=210ED24AF7431EDE0C0EA8225BA57DCD0C67D9EF5E948ED8' with link or location/anchor > 255 characters since it exceeds Excel's limit for URLS\n",
      "  force_unicode(url))\n"
     ]
    }
   ],
   "source": [
    "# A-3 输出excel档\n",
    "数据.to_excel(\"20春_Web数据挖掘_week05_sougou_weixin.xlsx\", sheet_name=\"搜查结果\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 拆分"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 文章链接"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 455,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['https://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6cwJThYulHEtVjXrGTiVgSwsS8AdK1eMIbKJrqUVRebxpQ4aYX0iWk1qXa8Fplpd97i_eAsv0X81DiwWew4H0AaNsMK1qfl8HkSWQqS3EiyzUFNg-DVu7GuC4PlMoY2SZ_ii9dfOFIMexihLz7kHYgkgX6LH4moKZc9L8tTe7P-EO50E8KbfZhLNJp3eVh_8iF-R_stCu8CFVtLsipj51Y1YKu-SkfyJu1UTGhkvmMcDI3OTRHYU2og..&type=2&query=python&token=161181389B20209DE6E046D7D1CD6454E666492D5E94BF4F',\n",
       " 'https://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6cwJThYulHEtVjXrGTiVgSwsS8AdK1eMIbKJrqUVRebxpQ4aYX0iWk1qXa8Fplpd9zeBZZku4K1O1kyaw8CKLILyXbe7mAXT-bAqhSEEBCgV6IWHP1R-3BC0oQu1losxIZyC9SY5utNXbNHiAS20VK-qimX6x_JIA6N9FubBW4KLQGjyyZyQ-HWLYI2RFgvGygyFUtdMVsBY2NZgbJovqw3pvBWjJskQLxFw33riXFU_yPfCoem7FzA..&type=2&query=python&token=161181389B20209DE6E046D7D1CD6454E666492D5E94BF4F',\n",
       " 'https://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6cwJThYulHEtVjXrGTiVgSwsS8AdK1eMIbKJrqUVRebxpQ4aYX0iWk1qXa8Fplpd9zeBZZku4K1O1kyaw8CKLILyXbe7mAXT-bAqhSEEBCgV6IWHP1R-3BAfW-gC2icbsb3fKwKK9nOLHSMVpC3HAvNGjcU29DkzoTwTnr3ZHXPjqf00t6wzW0AAgiOhGp4vN1bPB_cG9zafJt-xWw6ekPRJxP5YhDKSZlahmr0WvNmPdrgozfSg6bw..&type=2&query=python&token=161181389B20209DE6E046D7D1CD6454E666492D5E94BF4F',\n",
       " 'https://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6cwJThYulHEtVjXrGTiVgSwsS8AdK1eMIbKJrqUVRebxpQ4aYX0iWk1qXa8Fplpd9L_piby2ewl30syLxyvr2z8DoNCfiZION0Goz6_8IxLtxKskXw2E6blptbCW7aqDJcUOgba2ZElWBEc7KSU6smGzjZhjco8zdXchxyVD4cPZEUhMVUT1R2L9iBpSI9BKyxEdS0YheKIey_K-A8uAj2UxtQESqPuTykmaBJ3ZJxNMSYTqT4eyLmA..&type=2&query=python&token=161181389B20209DE6E046D7D1CD6454E666492D5E94BF4F',\n",
       " 'https://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6cwJThYulHEtVjXrGTiVgSwsS8AdK1eMIbKJrqUVRebxpQ4aYX0iWk1qXa8Fplpd9Fvwj93yYVupn1ld_mE71lrbE_ClOk7g5dG8LY3c-L10-aR8qb0eO0Ci_WtWoVNdr7wWiUWoGRsjajnQ_DYUiyNIQlQNLhXkOyPeE60vBfWKgsIG5bNTL5oDrfNVKehuUnu_bcIFAqogOkL5c3_kNprQQlvphyesS9AaQ_V88j53drgozfSg6bw..&type=2&query=python&token=161181389B20209DE6E046D7D1CD6454E666492D5E94BF4F',\n",
       " 'https://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6cwJThYulHEtVjXrGTiVgSwsS8AdK1eMIbKJrqUVRebxpQ4aYX0iWk1qXa8Fplpd9OYHH88O3fMmfFY9IEenrp9q-gd-NFFTXbzRWOEo4ES85xZ7oe9grv4aK9cCsfHYAmY6AEYfC3DWkAp9e_KQ0vPmf1bdk0xz6MSarCQtsrZ0yQuhsx9V9S6D4MLmcjx1rOvK_HukL4D9ywLtwZj1l1YBJS1HxEm29tWQcGv5FfLW00NVofL2iDw..&type=2&query=python&token=161181389B20209DE6E046D7D1CD6454E666492D5E94BF4F',\n",
       " 'https://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6cwJThYulHEtVjXrGTiVgSwsS8AdK1eMIbKJrqUVRebxpQ4aYX0iWk1qXa8Fplpd9Fvwj93yYVupn1ld_mE71lrbE_ClOk7g5dG8LY3c-L10-aR8qb0eO0G0XL7Y7Tpig1fUmSec9HiD8XLOPXHZ_U8r5LzAy9zCdNf0e1lAHMvD4YgO0sQZwsX2oTQX7H_ewh-HJOeVOiOlRLVURPvV5UUpxpu1BQxbkBUAJK_B2ROECn9lOoZLP2Q..&type=2&query=python&token=161181389B20209DE6E046D7D1CD6454E666492D5E94BF4F',\n",
       " 'https://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6cwJThYulHEtVjXrGTiVgSwsS8AdK1eMIbKJrqUVRebxpQ4aYX0iWk1qXa8Fplpd9drgI5mPVw6JtNk7xORe0Q_cXNL3Kfk3UVhmEsilx8-QGqTSgHLljX_WTU98ibb4YstQfG0bwZXsPkJGPsxfBmCy855KmqhXuZux1glSMLVvz1aMxuWZpLvztsiXBCBhZSQ00PUdk0wG3VRMKImCZD2rSo4UXM7beglhuRNq4VKc5eBgmN3LoYQ..&type=2&query=python&token=161181389B20209DE6E046D7D1CD6454E666492D5E94BF4F',\n",
       " 'https://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6cwJThYulHEtVjXrGTiVgSwsS8AdK1eMIbKJrqUVRebxpQ4aYX0iWk1qXa8Fplpd9lGe9t5CwY9LJNmk_V8YQ4giCPGaj6Y-TePe-uKXoEHiccT2QW2zlmyEGKEHA421d6TS4CC1OsmwU_pFgNaQ_speBz09lO9Te_a0639l3zyeqsuSqhSPL6uIm63Nrwsq1yMFTJSr-KwRl2iU9tNYFhQIbk4gBOjcxsX-H9uM_FLNhWLv6O878UA..&type=2&query=python&token=161181389B20209DE6E046D7D1CD6454E666492D5E94BF4F',\n",
       " 'https://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6cwJThYulHEtVjXrGTiVgSwsS8AdK1eMIbKJrqUVRebxpQ4aYX0iWk1qXa8Fplpd9Wo9SnCTLdbtag0iOKrWbUkpYvNm46SLhPnvNRbmO0ZXLgZ2UkD6x-MYwOTGq0hAu1yR_YebknZ5VvyjwxbwMGVjstyBxldvRaUMl0_WXiM5ktMfOwyocrdbQc89hxG-gxv0lzbibgWFA5htT53NHMqU6aGB_HGwPIhE2vBhq9b9flztgI7RZ1g..&type=2&query=python&token=161181389B20209DE6E046D7D1CD6454E666492D5E94BF4F']"
      ]
     },
     "execution_count": 455,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# A-4 文章链接 细化\n",
    "# href_list 文章链接\n",
    "article_href_list = []\n",
    "for item in r.html.xpath('//div[contains(@class,\"txt-box\")]/h3/a/@href'):\n",
    "#     print(item)\n",
    "    article_href_list.append(\"https://weixin.sogou.com\"+item)\n",
    "article_href_list"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 公众号链接"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 449,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['http://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqR9qmSowG7r2gBNPyrh3FVgwvDqyjOWdzeOCKnRJAxkUdZw5uyu3Hynw3YU8OWoUNYcaOf58YCgqXH6IVD2Cttu-1hfDLPAhT5AG8baJOzSK67Mho2fdeLHDctAPZVNzluOsufU76d3ec5RqUeByHO-O00efWrWmm&type=2&query=python&token=210ED24AF7431EDE0C0EA8225BA57DCD0C67D9EF5E948ED8',\n",
       " 'http://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqR9qmSowG7r2gBNPyrh3FVgwvDqyjOWdzeOCKnRJAxkUdZw5uyu3Hynw3YU8OWoUNYcaOf58YCgqXH6IVD2Cttu-1hfDLPAhT5AG8baJOzSK67Mho2fdeLHDctAPZVNzluOsufU76d3ec5RqUeByHO-O00efWrWmm&type=2&query=python&token=210ED24AF7431EDE0C0EA8225BA57DCD0C67D9EF5E948ED8',\n",
       " 'http://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqR9qmSowG7r2gBNPyrh3FVgwvDqyjOWdzeOCKnRJAxkUdZw5uyu3Hynw3YU8OWoUNYcaOf58YCgqXH6IVD2Cttu-1hfDLPAhT5AG8baJOzSK67Mho2fdeLHDctAPZVNzluOsufU76d3ec5RqUeByHO-O00efWrWmm&type=2&query=python&token=210ED24AF7431EDE0C0EA8225BA57DCD0C67D9EF5E948ED8',\n",
       " 'http://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqR9qmSowG7r2gBNPyrh3FVgwvDqyjOWdzeOCKnRJAxkUdZw5uyu3Hynw3YU8OWoUNYcaOf58YCgqXH6IVD2Cttu-1hfDLPAhT5AG8baJOzSK67Mho2fdeLHDctAPZVNzluOsufU76d3ec5RqUeByHO-O00efWrWmm&type=2&query=python&token=210ED24AF7431EDE0C0EA8225BA57DCD0C67D9EF5E948ED8',\n",
       " 'http://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqR9qmSowG7r2gBNPyrh3FVgwvDqyjOWdzeOCKnRJAxkUdZw5uyu3Hynw3YU8OWoUNYcaOf58YCgqXH6IVD2Cttu-1hfDLPAhT5AG8baJOzSK67Mho2fdeLHDctAPZVNzluOsufU76d3ec5RqUeByHO-O00efWrWmm&type=2&query=python&token=210ED24AF7431EDE0C0EA8225BA57DCD0C67D9EF5E948ED8',\n",
       " 'http://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqR9qmSowG7r2gBNPyrh3FVgwvDqyjOWdzeOCKnRJAxkUdZw5uyu3Hynw3YU8OWoUNYcaOf58YCgqXH6IVD2Cttu-1hfDLPAhT5AG8baJOzSK67Mho2fdeLHDctAPZVNzluOsufU76d3ec5RqUeByHO-O00efWrWmm&type=2&query=python&token=210ED24AF7431EDE0C0EA8225BA57DCD0C67D9EF5E948ED8',\n",
       " 'http://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqR9qmSowG7r2gBNPyrh3FVgwvDqyjOWdzeOCKnRJAxkUdZw5uyu3Hynw3YU8OWoUNYcaOf58YCgqXH6IVD2Cttu-1hfDLPAhT5AG8baJOzSK67Mho2fdeLHDctAPZVNzluOsufU76d3ec5RqUeByHO-O00efWrWmm&type=2&query=python&token=210ED24AF7431EDE0C0EA8225BA57DCD0C67D9EF5E948ED8',\n",
       " 'http://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqR9qmSowG7r2gBNPyrh3FVgwvDqyjOWdzeOCKnRJAxkUdZw5uyu3Hynw3YU8OWoUNYcaOf58YCgqXH6IVD2Cttu-1hfDLPAhT5AG8baJOzSK67Mho2fdeLHDctAPZVNzluOsufU76d3ec5RqUeByHO-O00efWrWmm&type=2&query=python&token=210ED24AF7431EDE0C0EA8225BA57DCD0C67D9EF5E948ED8',\n",
       " 'http://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqR9qmSowG7r2gBNPyrh3FVgwvDqyjOWdzeOCKnRJAxkUdZw5uyu3Hynw3YU8OWoUNYcaOf58YCgqXH6IVD2Cttu-1hfDLPAhT5AG8baJOzSK67Mho2fdeLHDctAPZVNzluOsufU76d3ec5RqUeByHO-O00efWrWmm&type=2&query=python&token=210ED24AF7431EDE0C0EA8225BA57DCD0C67D9EF5E948ED8',\n",
       " 'http://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqR9qmSowG7r2gBNPyrh3FVgwvDqyjOWdzeOCKnRJAxkUdZw5uyu3Hynw3YU8OWoUNYcaOf58YCgqXH6IVD2Cttu-1hfDLPAhT5AG8baJOzSK67Mho2fdeLHDctAPZVNzluOsufU76d3ec5RqUeByHO-O00efWrWmm&type=2&query=python&token=210ED24AF7431EDE0C0EA8225BA57DCD0C67D9EF5E948ED8']"
      ]
     },
     "execution_count": 449,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# A-5 公众号链接 细化\n",
    "[\"http://weixin.sogou.com\"+item\\\n",
    " for item in e.xpath('//div[contains(@class,\"txt-box\")]/div[@class=\"s-p\"]/a/@href')\\\n",
    " for e in 主要元素]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 来源和发表时间"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A-6 来源和发表时间 细化\n",
    "source = r.html.xpath('//div[contains(@class,\"txt-box\")]/div[@class=\"s-p\"]/a/text()')\n",
    "time =  r.html.xpath('//div[contains(@class,\"txt-box\")]/div[@class=\"s-p\"]/@t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 翻页：参数字典的拆解\n",
    "\n",
    "## xpath解析翻页a/@href"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 393,
   "metadata": {},
   "outputs": [],
   "source": [
    "# B-0   单一页面\n",
    "url = \"https://weixin.sogou.com/weixin\"\n",
    "session = HTMLSession()\n",
    "payload = {\n",
    "    \"query\":\"python\",\n",
    "    \"type\":\"2\"\n",
    "}\n",
    "\n",
    "r = session.get( url, params = payload)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 509,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<Element 'a' href='?query=python&type=2&page=2&ie=utf8' id='sogou_page_2' uigs='page_2'>, <Element 'a' href='?query=python&type=2&page=3&ie=utf8' id='sogou_page_3' uigs='page_3'>, <Element 'a' href='?query=python&type=2&page=4&ie=utf8' id='sogou_page_4' uigs='page_4'>, <Element 'a' href='?query=python&type=2&page=5&ie=utf8' id='sogou_page_5' uigs='page_5'>, <Element 'a' href='?query=python&type=2&page=6&ie=utf8' id='sogou_page_6' uigs='page_6'>, <Element 'a' href='?query=python&type=2&page=7&ie=utf8' id='sogou_page_7' uigs='page_7'>, <Element 'a' href='?query=python&type=2&page=8&ie=utf8' id='sogou_page_8' uigs='page_8'>, <Element 'a' href='?query=python&type=2&page=9&ie=utf8' id='sogou_page_9' uigs='page_9'>, <Element 'a' href='?query=python&type=2&page=10&ie=utf8' id='sogou_page_10' uigs='page_10'>, <Element 'a' class=('np',) href='?query=python&type=2&page=2&ie=utf8' id='sogou_next' uigs='page_next'>]\n",
      "['?query=python&type=2&page=2&ie=utf8', '?query=python&type=2&page=3&ie=utf8', '?query=python&type=2&page=4&ie=utf8', '?query=python&type=2&page=5&ie=utf8', '?query=python&type=2&page=6&ie=utf8', '?query=python&type=2&page=7&ie=utf8', '?query=python&type=2&page=8&ie=utf8', '?query=python&type=2&page=9&ie=utf8', '?query=python&type=2&page=10&ie=utf8', '?query=python&type=2&page=2&ie=utf8']\n",
      "{'2': '?query=python&type=2&page=2&ie=utf8', '3': '?query=python&type=2&page=3&ie=utf8', '4': '?query=python&type=2&page=4&ie=utf8', '5': '?query=python&type=2&page=5&ie=utf8', '6': '?query=python&type=2&page=6&ie=utf8', '7': '?query=python&type=2&page=7&ie=utf8', '8': '?query=python&type=2&page=8&ie=utf8', '9': '?query=python&type=2&page=9&ie=utf8', '10': '?query=python&type=2&page=10&ie=utf8', '下一页': '?query=python&type=2&page=2&ie=utf8'}\n"
     ]
    }
   ],
   "source": [
    "# A-1  xpath 解析翻页a/@href\n",
    "xpath_翻页a = '//div[@class=\"p-fy\"]/a'\n",
    "xpath_翻页a = '//div[@class=\"p-fy\"]/a'\n",
    "print (r.html.xpath(xpath_翻页a))\n",
    "\n",
    "href_列表 = [x.xpath('//@href')[0] for x in r.html.xpath(xpath_翻页a)]\n",
    "print (href_列表)\n",
    "\n",
    "href_字典 = {x.text:x.xpath('//@href')[0]  for x in r.html.xpath(xpath_翻页a)}\n",
    "print (href_字典)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 511,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'query': ['python'], 'type': ['2'], 'page': ['2'], 'ie': ['utf8']}"
      ]
     },
     "execution_count": 511,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "parse_qs(\"query=python&type=2&page=2&ie=utf8\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 395,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>scheme</th>\n",
       "      <th>netloc</th>\n",
       "      <th>path</th>\n",
       "      <th>params</th>\n",
       "      <th>query</th>\n",
       "      <th>fragment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>query=python&amp;type=2&amp;page=2&amp;ie=utf8</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>query=python&amp;type=2&amp;page=3&amp;ie=utf8</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>query=python&amp;type=2&amp;page=4&amp;ie=utf8</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>query=python&amp;type=2&amp;page=5&amp;ie=utf8</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>query=python&amp;type=2&amp;page=6&amp;ie=utf8</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>query=python&amp;type=2&amp;page=7&amp;ie=utf8</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>query=python&amp;type=2&amp;page=8&amp;ie=utf8</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>query=python&amp;type=2&amp;page=9&amp;ie=utf8</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>query=python&amp;type=2&amp;page=10&amp;ie=utf8</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>query=python&amp;type=2&amp;page=2&amp;ie=utf8</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  scheme netloc path params                                query fragment\n",
       "0                             query=python&type=2&page=2&ie=utf8         \n",
       "1                             query=python&type=2&page=3&ie=utf8         \n",
       "2                             query=python&type=2&page=4&ie=utf8         \n",
       "3                             query=python&type=2&page=5&ie=utf8         \n",
       "4                             query=python&type=2&page=6&ie=utf8         \n",
       "5                             query=python&type=2&page=7&ie=utf8         \n",
       "6                             query=python&type=2&page=8&ie=utf8         \n",
       "7                             query=python&type=2&page=9&ie=utf8         \n",
       "8                            query=python&type=2&page=10&ie=utf8         \n",
       "9                             query=python&type=2&page=2&ie=utf8         "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "scheme      1\n",
      "netloc      1\n",
      "path        1\n",
      "params      1\n",
      "query       9\n",
      "fragment    1\n",
      "dtype: int64\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>query</th>\n",
       "      <th>type</th>\n",
       "      <th>page</th>\n",
       "      <th>ie</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>python</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>utf8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>python</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>utf8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>python</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>utf8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>python</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>utf8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>python</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>utf8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>python</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>utf8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>python</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "      <td>utf8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>python</td>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "      <td>utf8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>python</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>utf8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>python</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>utf8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    query type page    ie\n",
       "0  python    2    2  utf8\n",
       "1  python    2    3  utf8\n",
       "2  python    2    4  utf8\n",
       "3  python    2    5  utf8\n",
       "4  python    2    6  utf8\n",
       "5  python    2    7  utf8\n",
       "6  python    2    8  utf8\n",
       "7  python    2    9  utf8\n",
       "8  python    2   10  utf8\n",
       "9  python    2    2  utf8"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "query    1\n",
      "type     1\n",
      "page     9\n",
      "ie       1\n",
      "dtype: int64\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>query</th>\n",
       "      <th>type</th>\n",
       "      <th>page</th>\n",
       "      <th>ie</th>\n",
       "      <th>page_int</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>python</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>utf8</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>python</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>utf8</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>python</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>utf8</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>python</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>utf8</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>python</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>utf8</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>python</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>utf8</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>python</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "      <td>utf8</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>python</td>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "      <td>utf8</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>python</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>utf8</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>python</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>utf8</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    query type page    ie  page_int\n",
       "0  python    2    2  utf8         2\n",
       "1  python    2    3  utf8         3\n",
       "2  python    2    4  utf8         4\n",
       "3  python    2    5  utf8         5\n",
       "4  python    2    6  utf8         6\n",
       "5  python    2    7  utf8         7\n",
       "6  python    2    8  utf8         8\n",
       "7  python    2    9  utf8         9\n",
       "8  python    2   10  utf8        10\n",
       "9  python    2    2  utf8         2"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# A-2 建构参数模板：找到关键参数及参数结构\n",
    "\n",
    "# 需要模组库\n",
    "from urllib.parse import urlparse, parse_qs\n",
    "import pandas as pd\n",
    "from IPython.display import display, HTML\n",
    "\n",
    "\n",
    "\n",
    "df = pd.DataFrame([ urlparse(x) for x in href_列表])\n",
    "\n",
    "df_qs = pd.DataFrame([{k:v[0] for k,v in parse_qs(x).items()} for x in df['query'] ])\n",
    "\n",
    "display(df)\n",
    "\n",
    "\n",
    "\n",
    "print(df.nunique())\n",
    "display(df_qs)\n",
    "print(df_qs.nunique())\n",
    "\n",
    "df_qs.page\n",
    "df_qs = df_qs.assign (page_int=df_qs.page.astype(int)) # 变成整数(数值)\n",
    "display(df_qs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 512,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'query': ['python'], 'type': ['2'], 'page': ['4'], 'ie': ['utf8']}\n",
      "{'2': '?query=python&type=2&page=2&ie=utf8', '3': '?query=python&type=2&page=3&ie=utf8', '4': '?query=python&type=2&page=4&ie=utf8', '5': '?query=python&type=2&page=5&ie=utf8', '6': '?query=python&type=2&page=6&ie=utf8', '7': '?query=python&type=2&page=7&ie=utf8', '8': '?query=python&type=2&page=8&ie=utf8', '9': '?query=python&type=2&page=9&ie=utf8', '10': '?query=python&type=2&page=10&ie=utf8', '下一页': '?query=python&type=2&page=2&ie=utf8'}\n"
     ]
    }
   ],
   "source": [
    "# A-2 建构参数模板：找到关键参数及参数结构\n",
    "\n",
    "def parse_url_qs_for_page (url):\n",
    "    six_parts = urlparse(url) \n",
    "    out = parse_qs(six_parts.query)\n",
    "    return (out)\n",
    "\n",
    "# 取一例做模板\n",
    "参数模板 = parse_url_qs_for_page(href_列表[2])\n",
    "print (参数模板)\n",
    "\n",
    "# 重新审视：href_字典\n",
    "print (href_字典)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 513,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\n",
      "10\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{1: {'query': ['用户体验'], 'type': ['2'], 'page': [1], 'ie': ['utf8']},\n",
       " 2: {'query': ['用户体验'], 'type': ['2'], 'page': [2], 'ie': ['utf8']},\n",
       " 3: {'query': ['用户体验'], 'type': ['2'], 'page': [3], 'ie': ['utf8']},\n",
       " 4: {'query': ['用户体验'], 'type': ['2'], 'page': [4], 'ie': ['utf8']},\n",
       " 5: {'query': ['用户体验'], 'type': ['2'], 'page': [5], 'ie': ['utf8']},\n",
       " 6: {'query': ['用户体验'], 'type': ['2'], 'page': [6], 'ie': ['utf8']},\n",
       " 7: {'query': ['用户体验'], 'type': ['2'], 'page': [7], 'ie': ['utf8']},\n",
       " 8: {'query': ['用户体验'], 'type': ['2'], 'page': [8], 'ie': ['utf8']},\n",
       " 9: {'query': ['用户体验'], 'type': ['2'], 'page': [9], 'ie': ['utf8']},\n",
       " 10: {'query': ['用户体验'], 'type': ['2'], 'page': [10], 'ie': ['utf8']}}"
      ]
     },
     "execution_count": 513,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# A-3 建构参数模板生成器：query（搜索关键字）page\n",
    "def 参数模板生成(query, page):\n",
    "    参数 = 参数模板.copy()\n",
    "    参数['page'] = page\n",
    "    参数['query'] = query\n",
    "    return (参数)\n",
    "\n",
    "# -----------    参数模版生成器 目的 构建payload\n",
    "# payload={\n",
    "    \n",
    "# }\n",
    "参数_query_用户体验_page = { \n",
    "    i:参数模板生成(page = [i], \\\n",
    "                  query = ['用户体验']) \\\n",
    "    for i,v in href_字典.items()\\\n",
    "    }\n",
    "\n",
    "print (df_qs.page_int.min()) # 最小值只有2 1的时候没有链接？\n",
    "print (df_qs.page_int.max()) # 最大值只有9\n",
    "\n",
    "参数_query_用户体验_page = { \n",
    "    i:参数模板生成(page = [i], \\\n",
    "                  query = ['用户体验']) \\\n",
    "    for i in range(df_qs.page_int.min()-1,df_qs.page_int.max()+1)\\\n",
    "    }\n",
    "参数_query_用户体验_page"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 封装request_sougou_weixin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 534,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "第 1 页\n",
      "payload: {'query': 'python', 'type': '2', 'page': 1}\n",
      "url: https://weixin.sogou.com/weixin\n",
      "第 1 页 的Dataframe\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>手把手教你用 Python 绘制酷炫的桑基图!</td>\n",
       "      <td>Python手把手绘制桑基图动手之前,我们再次敲黑板,回顾桑基图组成要素的重点——节点、边和...</td>\n",
       "      <td>CSDN</td>\n",
       "      <td>1585706777</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-01 02:06:17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Python 十大装 B 语法</td>\n",
       "      <td>作者 | 许向武责编 | 郭芮出品 | CSDN 博客Python 是一种代表简单思想的语言...</td>\n",
       "      <td>CSDN</td>\n",
       "      <td>1572764522</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2019-11-03 07:02:02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>找对象那么难?python s3cmd 帮你解决</td>\n",
       "      <td>python 的第三方库的安装目录:这个库实现 S3 协议的封装,解析,发包,收包等操作命令...</td>\n",
       "      <td>Python程序员</td>\n",
       "      <td>1586823024</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-14 00:10:24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9.9元Python小课体验课,限100人!</td>\n",
       "      <td>Python几乎正逐渐成为,未来5~10年新一代“Office技能”.01初入职场如何不被轻...</td>\n",
       "      <td>大学生必备网</td>\n",
       "      <td>1586827802</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-14 01:30:02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>为什么 Python 不是未来的编程语言?</td>\n",
       "      <td>作者丨Rhea Moutafis译者丨刘志勇策划丨刘燕Python 现在如此火爆,已发展成为...</td>\n",
       "      <td>InfoQ</td>\n",
       "      <td>1586824246</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-14 00:30:46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>小白怎样学python,才不会从入门到放弃?</td>\n",
       "      <td>Python已进入小学教材将加入高考编程可以将我们从重复的工作中解放出来,去完成更多更有创造...</td>\n",
       "      <td>业余无线电</td>\n",
       "      <td>1586826425</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-14 01:07:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7 款 Python 开源框架的优劣总结</td>\n",
       "      <td>下面就简单的介绍一下Python七款开源框架的优劣.DjangoDjango 应该是最出名的...</td>\n",
       "      <td>Python那些事</td>\n",
       "      <td>1586823354</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-14 00:15:54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Python绘制傅里叶变换、反变换与带通滤波图像</td>\n",
       "      <td>推荐图书:《Python可以这样学》,ISBN:9787302456469,董付国,清华大学...</td>\n",
       "      <td>Python小屋</td>\n",
       "      <td>1586818537</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-13 22:55:37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>跟李老师学编程-Python3 错误和异常</td>\n",
       "      <td>作为 Python 初学者,在刚学习 Python 编程时,经常会看到一些报错信息,在前面我...</td>\n",
       "      <td>极客青少编程</td>\n",
       "      <td>1586826032</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-14 01:00:32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>用Python模拟新冠肺炎的传播</td>\n",
       "      <td>我想到的是单纯地在Python中使用可视化工具matplotlib来编写这个模拟过程.但是对...</td>\n",
       "      <td>Python程序员</td>\n",
       "      <td>1586823024</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-14 00:10:24</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         标题  \\\n",
       "0   手把手教你用 Python 绘制酷炫的桑基图!   \n",
       "1           Python 十大装 B 语法   \n",
       "2  找对象那么难?python s3cmd 帮你解决   \n",
       "3    9.9元Python小课体验课,限100人!   \n",
       "4     为什么 Python 不是未来的编程语言?   \n",
       "5    小白怎样学python,才不会从入门到放弃?   \n",
       "6      7 款 Python 开源框架的优劣总结   \n",
       "7  Python绘制傅里叶变换、反变换与带通滤波图像   \n",
       "8     跟李老师学编程-Python3 错误和异常   \n",
       "9          用Python模拟新冠肺炎的传播   \n",
       "\n",
       "                                                  摘要     source          时间  \\\n",
       "0  Python手把手绘制桑基图动手之前,我们再次敲黑板,回顾桑基图组成要素的重点——节点、边和...       CSDN  1585706777   \n",
       "1  作者 | 许向武责编 | 郭芮出品 | CSDN 博客Python 是一种代表简单思想的语言...       CSDN  1572764522   \n",
       "2  python 的第三方库的安装目录:这个库实现 S3 协议的封装,解析,发包,收包等操作命令...  Python程序员  1586823024   \n",
       "3  Python几乎正逐渐成为,未来5~10年新一代“Office技能”.01初入职场如何不被轻...     大学生必备网  1586827802   \n",
       "4  作者丨Rhea Moutafis译者丨刘志勇策划丨刘燕Python 现在如此火爆,已发展成为...      InfoQ  1586824246   \n",
       "5  Python已进入小学教材将加入高考编程可以将我们从重复的工作中解放出来,去完成更多更有创造...      业余无线电  1586826425   \n",
       "6  下面就简单的介绍一下Python七款开源框架的优劣.DjangoDjango 应该是最出名的...  Python那些事  1586823354   \n",
       "7  推荐图书:《Python可以这样学》,ISBN:9787302456469,董付国,清华大学...   Python小屋  1586818537   \n",
       "8  作为 Python 初学者,在刚学习 Python 编程时,经常会看到一些报错信息,在前面我...     极客青少编程  1586826032   \n",
       "9  我想到的是单纯地在Python中使用可视化工具matplotlib来编写这个模拟过程.但是对...  Python程序员  1586823024   \n",
       "\n",
       "                                               公众号链接  \\\n",
       "0  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "1  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "2  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "3  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "4  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "5  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "6  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "7  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "8  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "9  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "\n",
       "                                                文章链接                格式化时间  \n",
       "0  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-01 02:06:17  \n",
       "1  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2019-11-03 07:02:02  \n",
       "2  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-14 00:10:24  \n",
       "3  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-14 01:30:02  \n",
       "4  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-14 00:30:46  \n",
       "5  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-14 01:07:05  \n",
       "6  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-14 00:15:54  \n",
       "7  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-13 22:55:37  \n",
       "8  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-14 01:00:32  \n",
       "9  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-14 00:10:24  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "第 2 页\n",
      "payload: {'query': 'python', 'type': '2', 'page': 2}\n",
      "url: https://weixin.sogou.com/weixin\n",
      "第 2 页 的Dataframe\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>MySQL与Python交互入门</td>\n",
       "      <td>(0, ＂python1＂, 50), (0, ＂python2＂, 60), (0, ＂p...</td>\n",
       "      <td>小詹学Python</td>\n",
       "      <td>1586824285</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-14 00:31:25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Python到底是什么?学姐靠它拿了5个offer</td>\n",
       "      <td>“熟悉Python编程优先”这样的条件.需要的不是埋头苦干而是一次实现你需要的不是埋头苦干而...</td>\n",
       "      <td>遇见百种人生</td>\n",
       "      <td>1586790385</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-13 15:06:25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>python进阶-函数式编程(高级函数)</td>\n",
       "      <td>python sorted(iterable, key=None, reverse=Fals...</td>\n",
       "      <td>StriveNow</td>\n",
       "      <td>1586799046</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-13 17:30:46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Python中 Pandas 50题冲关</td>\n",
       "      <td>Python中的Numpy基础20问参考资料 | 100-pandas-puzzles - ...</td>\n",
       "      <td>Python那些事</td>\n",
       "      <td>1586823354</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-14 00:15:54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Python内存管理机制</td>\n",
       "      <td>在之前的面试中有被问到Python的内存管理机制,回答的不好,之后查阅了一些文章,这里简单做...</td>\n",
       "      <td>Begin Now</td>\n",
       "      <td>1586822446</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-14 00:00:46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Python小课教程:模块</td>\n",
       "      <td>在Python中,一个.py文件就称之为一个模块(Module).使用模块有什么好处?最大的...</td>\n",
       "      <td>Python小课教程</td>\n",
       "      <td>1586793485</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-13 15:58:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Python 碾压 Excel</td>\n",
       "      <td>根据最新消息,Python成了各大公司的心头好投行、券商、咨询、四大纷纷表示:想要顺利求职,...</td>\n",
       "      <td>证经学社</td>\n",
       "      <td>1586825623</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-14 00:53:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>势头强劲的 Python PK 强大的 C++,究竟谁更胜一筹?</td>\n",
       "      <td>1.如果在Windows机器上用Windows编译器来编译代码,则会生成Windows机器代...</td>\n",
       "      <td>CSDN</td>\n",
       "      <td>1580544534</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-02-01 08:08:54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Python 什么时候会被取代?</td>\n",
       "      <td>为什么Python不是未来的编程语言?作者 | Rhea Moutafis译者 | 弯月,责...</td>\n",
       "      <td>CSDN</td>\n",
       "      <td>1586052093</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-05 02:01:33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>没错!Python杀死了Excel!</td>\n",
       "      <td>Python数据分析有多强大?举个栗子:用几行代码就能快速整理数据并出图:甚至可以编写合成多...</td>\n",
       "      <td>CSDN</td>\n",
       "      <td>1583654301</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-03-08 07:58:21</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 标题  \\\n",
       "0                  MySQL与Python交互入门   \n",
       "1         Python到底是什么?学姐靠它拿了5个offer   \n",
       "2              python进阶-函数式编程(高级函数)   \n",
       "3              Python中 Pandas 50题冲关   \n",
       "4                      Python内存管理机制   \n",
       "5                     Python小课教程:模块   \n",
       "6                   Python 碾压 Excel   \n",
       "7  势头强劲的 Python PK 强大的 C++,究竟谁更胜一筹?   \n",
       "8                  Python 什么时候会被取代?   \n",
       "9                没错!Python杀死了Excel!   \n",
       "\n",
       "                                                  摘要      source          时间  \\\n",
       "0  (0, ＂python1＂, 50), (0, ＂python2＂, 60), (0, ＂p...   小詹学Python  1586824285   \n",
       "1  “熟悉Python编程优先”这样的条件.需要的不是埋头苦干而是一次实现你需要的不是埋头苦干而...      遇见百种人生  1586790385   \n",
       "2  python sorted(iterable, key=None, reverse=Fals...   StriveNow  1586799046   \n",
       "3  Python中的Numpy基础20问参考资料 | 100-pandas-puzzles - ...   Python那些事  1586823354   \n",
       "4  在之前的面试中有被问到Python的内存管理机制,回答的不好,之后查阅了一些文章,这里简单做...   Begin Now  1586822446   \n",
       "5  在Python中,一个.py文件就称之为一个模块(Module).使用模块有什么好处?最大的...  Python小课教程  1586793485   \n",
       "6  根据最新消息,Python成了各大公司的心头好投行、券商、咨询、四大纷纷表示:想要顺利求职,...        证经学社  1586825623   \n",
       "7  1.如果在Windows机器上用Windows编译器来编译代码,则会生成Windows机器代...        CSDN  1580544534   \n",
       "8  为什么Python不是未来的编程语言?作者 | Rhea Moutafis译者 | 弯月,责...        CSDN  1586052093   \n",
       "9  Python数据分析有多强大?举个栗子:用几行代码就能快速整理数据并出图:甚至可以编写合成多...        CSDN  1583654301   \n",
       "\n",
       "                                               公众号链接  \\\n",
       "0  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "1  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "2  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "3  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "4  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "5  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "6  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "7  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "8  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "9  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "\n",
       "                                                文章链接                格式化时间  \n",
       "0  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-14 00:31:25  \n",
       "1  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-13 15:06:25  \n",
       "2  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-13 17:30:46  \n",
       "3  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-14 00:15:54  \n",
       "4  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-14 00:00:46  \n",
       "5  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-13 15:58:05  \n",
       "6  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-14 00:53:43  \n",
       "7  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-02-01 08:08:54  \n",
       "8  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-05 02:01:33  \n",
       "9  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-03-08 07:58:21  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "第 3 页\n",
      "payload: {'query': 'python', 'type': '2', 'page': 3}\n",
      "url: https://weixin.sogou.com/weixin\n",
      "第 3 页 的Dataframe\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "第 4 页\n",
      "payload: {'query': 'python', 'type': '2', 'page': 4}\n",
      "url: https://weixin.sogou.com/weixin\n",
      "第 4 页 的Dataframe\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "第 5 页\n",
      "payload: {'query': 'python', 'type': '2', 'page': 5}\n",
      "url: https://weixin.sogou.com/weixin\n",
      "第 5 页 的Dataframe\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "第 6 页\n",
      "payload: {'query': 'python', 'type': '2', 'page': 6}\n",
      "url: https://weixin.sogou.com/weixin\n",
      "第 6 页 的Dataframe\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "第 7 页\n",
      "payload: {'query': 'python', 'type': '2', 'page': 7}\n",
      "url: https://weixin.sogou.com/weixin\n",
      "第 7 页 的Dataframe\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "第 8 页\n",
      "payload: {'query': 'python', 'type': '2', 'page': 8}\n",
      "url: https://weixin.sogou.com/weixin\n",
      "第 8 页 的Dataframe\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "第 9 页\n",
      "payload: {'query': 'python', 'type': '2', 'page': 9}\n",
      "url: https://weixin.sogou.com/weixin\n",
      "第 9 页 的Dataframe\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/xuzhichao/anaconda3/lib/python3.7/site-packages/xlsxwriter/worksheet.py:910: UserWarning: Ignoring URL 'https://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqR9qmSowG7r2gBNPyrh3FVgwvDqyjOWdzeOCKnRJAxkUdZw5uyu3Hynw3YU8OWoUNYcaOf58YCgqXH6IVD2Cttu-1hfDLPAhT5AG8baJOzSK67Mho2fdeLHDctAPZVNzluOsufU76d3ec5RqUeByHO-O00efWrWmm&type=2&query=python&token=210ED24AF7431EDE0C0EA8225BA57DCD0C67D9EF5E948ED8' with link or location/anchor > 255 characters since it exceeds Excel's limit for URLS\n",
      "  force_unicode(url))\n",
      "/Users/xuzhichao/anaconda3/lib/python3.7/site-packages/xlsxwriter/worksheet.py:910: UserWarning: Ignoring URL 'https://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6cwJThYulHEtVjXrGTiVgS9dZesVPC425bopQRjCb2BlpQ4aYX0iWk1qXa8Fplpd9Hhup-WCiA6jKWLk-8yqbp2an3quTv2SP3gDmUfMBWk4hAl2leQHjMIc7a1X13FuuMHylJLJweSgqKYEhdkYntOId8_j3Lu1YmkUd6oG33m7wXpIptlAt7U6T7RdGxQgLBllaLKbmvInP_dLDClI7biNRhieJZTrlVKy4Eeemcj_f3q8uRDTfkw..&type=2&query=python&token=210ED24AF7431EDE0C0EA8225BA57DCD0C67D9EF5E948ED8' with link or location/anchor > 255 characters since it exceeds Excel's limit for URLS\n",
      "  force_unicode(url))\n"
     ]
    }
   ],
   "source": [
    "session = HTMLSession()\n",
    "def request_sougou_weixin(url,params):\n",
    "   \n",
    "    r = session.get( url, params = payload, )\n",
    "    \n",
    "    \n",
    "    主要元素 =  r.html.xpath( '//ul[@class=\"news-list\"]/li')\n",
    "\n",
    "    dict_xpaths = {\n",
    "        'text':{\n",
    "            'source':'//div[contains(@class,\"txt-box\")]/div[@class=\"s-p\"]/a/text()',\n",
    "            '时间':'//div[contains(@class,\"txt-box\")]/div[@class=\"s-p\"]/@t',       \n",
    "        },\n",
    "        'text_content':{\n",
    "            '标题':'//div[contains(@class,\"txt-box\")]/h3/a',\n",
    "            '摘要':'//div[contains(@class,\"txt-box\")]/p'\n",
    "        },\n",
    "        'href':{\n",
    "            \"公众号链接\":'//div[contains(@class,\"txt-box\")]/div[@class=\"s-p\"]/a/@href',\n",
    "            \"文章链接\":'//div[contains(@class,\"txt-box\")]/h3/a/@href',\n",
    "        }\n",
    "    }\n",
    "\n",
    "\n",
    "    def get_e_text_content(_xpath_):\n",
    "        # 高级列表推导\n",
    "        暂存结果 = [e.xpath(_xpath_)[0].lxml.text_content() for e in 主要元素]\n",
    "\n",
    "        return(暂存结果)\n",
    "\n",
    "    def get_e_text(_xpath_):\n",
    "        暂存结果 = [\"\".join([x.strip() if type(x) is str else x.text.strip() for x in e.xpath(_xpath_)]) for e in 主要元素]\n",
    "        return(暂存结果)\n",
    "\n",
    "    def get_e_href(_xpath_):\n",
    "        # 高级列表推导\n",
    "        暂存结果 = [\"https://weixin.sogou.com\"+item\\\n",
    "                for item in e.xpath(_xpath_)\\\n",
    "                for e in 主要元素]\n",
    "        return(暂存结果)\n",
    "\n",
    "    数据字典 = dict()\n",
    "    数据字典 = {k:get_e_text_content(v) for k,v in dict_xpaths['text_content'].items()}\n",
    "\n",
    "    数据字典.update({k:get_e_text(v) for k,v in dict_xpaths['text'].items()})\n",
    "    数据字典.update({k:get_e_href(v) for k,v in dict_xpaths['href'].items()})\n",
    "\n",
    "    数据 = pd.DataFrame(数据字典)\n",
    "    数据 = 数据.assign(\n",
    "        格式化时间 = lambda x: pd.to_datetime(x[\"时间\"], unit='s').dt.strftime('%Y-%m-%d %H:%M:%S')\n",
    "    )\n",
    "    return(数据)\n",
    "\n",
    "\n",
    "url = \"https://weixin.sogou.com/weixin\"\n",
    "\n",
    "\n",
    "list_df=[]\n",
    "for i in range(1,10):\n",
    "    print(\"第\",i,\"页\")\n",
    "    payload = {\n",
    "        \"query\":\"python\",\n",
    "        \"type\":\"2\",\n",
    "        \"page\":i\n",
    "\n",
    "    } \n",
    "    print(\"payload:\",payload)\n",
    "    print(\"url:\",url)\n",
    "    time.sleep(10+10*random())\n",
    "    df = request_sougou_weixin(url,params=payload)\n",
    "    print(\"第\",i,\"页\",\"的Dataframe\")\n",
    "    display(df)\n",
    "    \n",
    "    df.to_csv(\"20春_Web数据挖掘_week05_sougou_weixin_{key}_{k}.tsv\"\\\n",
    "                  .format(key=\"python\", k=i), sep=\"\\t\", encoding=\"utf8\")\n",
    "        \n",
    "    df = df.assign (query = key)  # 区分  query   \n",
    "    df = df.assign (page = i)  # 区分  page    \n",
    "    list_df.append(df)\n",
    "    \n",
    "\n",
    "df_all = pd.concat(list_df).reset_index()\n",
    "df_all.index.name = '序'\n",
    "\n",
    "df_all.to_excel(\"20春_Web数据挖掘_week05_sougou_weixin_翻页_5.xlsx\",\\\n",
    "                sheet_name=\"_\".join(query))\n",
    "# 预估时间: 4*5秒*10 =200\n",
    "# 预估数量: 4*40*10 =1600"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "r.request"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 多个页面爬取"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 514,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'query': ['python'], 'type': ['2'], 'page': ['1'], 'ie': ['utf8']}\n",
      "  scheme netloc path params                                query fragment\n",
      "0                             query=python&type=2&page=2&ie=utf8         \n",
      "1                             query=python&type=2&page=3&ie=utf8         \n",
      "2                             query=python&type=2&page=4&ie=utf8         \n",
      "3                             query=python&type=2&page=5&ie=utf8         \n",
      "4                             query=python&type=2&page=6&ie=utf8         \n",
      "5                             query=python&type=2&page=7&ie=utf8         \n",
      "6                             query=python&type=2&page=8&ie=utf8         \n",
      "7                             query=python&type=2&page=9&ie=utf8         \n",
      "8                            query=python&type=2&page=10&ie=utf8         \n",
      "9                             query=python&type=2&page=2&ie=utf8         \n",
      "python 10\n",
      "参数_query_X_page {1: {'query': ['python'], 'type': ['2'], 'page': [1], 'ie': ['utf8']}, 2: {'query': ['python'], 'type': ['2'], 'page': [2], 'ie': ['utf8']}, 3: {'query': ['python'], 'type': ['2'], 'page': [3], 'ie': ['utf8']}, 4: {'query': ['python'], 'type': ['2'], 'page': [4], 'ie': ['utf8']}, 5: {'query': ['python'], 'type': ['2'], 'page': [5], 'ie': ['utf8']}, 6: {'query': ['python'], 'type': ['2'], 'page': [6], 'ie': ['utf8']}, 7: {'query': ['python'], 'type': ['2'], 'page': [7], 'ie': ['utf8']}, 8: {'query': ['python'], 'type': ['2'], 'page': [8], 'ie': ['utf8']}, 9: {'query': ['python'], 'type': ['2'], 'page': [9], 'ie': ['utf8']}}\n",
      "payload {'query': ['python'], 'type': ['2'], 'page': [1], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>手把手教你用 Python 绘制酷炫的桑基图!</td>\n",
       "      <td>Python手把手绘制桑基图动手之前,我们再次敲黑板,回顾桑基图组成要素的重点——节点、边和...</td>\n",
       "      <td>CSDN</td>\n",
       "      <td>1585706777</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-01 02:06:17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Python 十大装 B 语法</td>\n",
       "      <td>作者 | 许向武责编 | 郭芮出品 | CSDN 博客Python 是一种代表简单思想的语言...</td>\n",
       "      <td>CSDN</td>\n",
       "      <td>1572764522</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2019-11-03 07:02:02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>找对象那么难?python s3cmd 帮你解决</td>\n",
       "      <td>python 的第三方库的安装目录:这个库实现 S3 协议的封装,解析,发包,收包等操作命令...</td>\n",
       "      <td>Python程序员</td>\n",
       "      <td>1586823024</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-14 00:10:24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7 款 Python 开源框架的优劣总结</td>\n",
       "      <td>下面就简单的介绍一下Python七款开源框架的优劣.DjangoDjango 应该是最出名的...</td>\n",
       "      <td>Python那些事</td>\n",
       "      <td>1586823354</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-14 00:15:54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>为什么 Python 不是未来的编程语言?</td>\n",
       "      <td>作者丨Rhea Moutafis译者丨刘志勇策划丨刘燕Python 现在如此火爆,已发展成为...</td>\n",
       "      <td>InfoQ</td>\n",
       "      <td>1586824246</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-14 00:30:46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Python绘制傅里叶变换、反变换与带通滤波图像</td>\n",
       "      <td>推荐图书:《Python可以这样学》,ISBN:9787302456469,董付国,清华大学...</td>\n",
       "      <td>Python小屋</td>\n",
       "      <td>1586818537</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-13 22:55:37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>MySQL与Python交互入门</td>\n",
       "      <td>(0, ＂python1＂, 50), (0, ＂python2＂, 60), (0, ＂p...</td>\n",
       "      <td>小詹学Python</td>\n",
       "      <td>1586824285</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-14 00:31:25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>python进阶-函数式编程(高级函数)</td>\n",
       "      <td>python sorted(iterable, key=None, reverse=Fals...</td>\n",
       "      <td>StriveNow</td>\n",
       "      <td>1586799046</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-13 17:30:46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>用Python模拟新冠肺炎的传播</td>\n",
       "      <td>我想到的是单纯地在Python中使用可视化工具matplotlib来编写这个模拟过程.但是对...</td>\n",
       "      <td>Python程序员</td>\n",
       "      <td>1586823024</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-14 00:10:24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>python_pytest基础知识1:</td>\n",
       "      <td>1、python字典基础使用:dl = { ＂a＂ : None, ＂b＂ : 11, ＂c...</td>\n",
       "      <td>AdminWiki</td>\n",
       "      <td>1586797342</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-13 17:02:22</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         标题  \\\n",
       "0   手把手教你用 Python 绘制酷炫的桑基图!   \n",
       "1           Python 十大装 B 语法   \n",
       "2  找对象那么难?python s3cmd 帮你解决   \n",
       "3      7 款 Python 开源框架的优劣总结   \n",
       "4     为什么 Python 不是未来的编程语言?   \n",
       "5  Python绘制傅里叶变换、反变换与带通滤波图像   \n",
       "6          MySQL与Python交互入门   \n",
       "7      python进阶-函数式编程(高级函数)   \n",
       "8          用Python模拟新冠肺炎的传播   \n",
       "9       python_pytest基础知识1:   \n",
       "\n",
       "                                                  摘要     source          时间  \\\n",
       "0  Python手把手绘制桑基图动手之前,我们再次敲黑板,回顾桑基图组成要素的重点——节点、边和...       CSDN  1585706777   \n",
       "1  作者 | 许向武责编 | 郭芮出品 | CSDN 博客Python 是一种代表简单思想的语言...       CSDN  1572764522   \n",
       "2  python 的第三方库的安装目录:这个库实现 S3 协议的封装,解析,发包,收包等操作命令...  Python程序员  1586823024   \n",
       "3  下面就简单的介绍一下Python七款开源框架的优劣.DjangoDjango 应该是最出名的...  Python那些事  1586823354   \n",
       "4  作者丨Rhea Moutafis译者丨刘志勇策划丨刘燕Python 现在如此火爆,已发展成为...      InfoQ  1586824246   \n",
       "5  推荐图书:《Python可以这样学》,ISBN:9787302456469,董付国,清华大学...   Python小屋  1586818537   \n",
       "6  (0, ＂python1＂, 50), (0, ＂python2＂, 60), (0, ＂p...  小詹学Python  1586824285   \n",
       "7  python sorted(iterable, key=None, reverse=Fals...  StriveNow  1586799046   \n",
       "8  我想到的是单纯地在Python中使用可视化工具matplotlib来编写这个模拟过程.但是对...  Python程序员  1586823024   \n",
       "9  1、python字典基础使用:dl = { ＂a＂ : None, ＂b＂ : 11, ＂c...  AdminWiki  1586797342   \n",
       "\n",
       "                                               公众号链接  \\\n",
       "0  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "1  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "2  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "3  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "4  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "5  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "6  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "7  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "8  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "9  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "\n",
       "                                                文章链接                格式化时间  \n",
       "0  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-01 02:06:17  \n",
       "1  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2019-11-03 07:02:02  \n",
       "2  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-14 00:10:24  \n",
       "3  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-14 00:15:54  \n",
       "4  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-14 00:30:46  \n",
       "5  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-13 22:55:37  \n",
       "6  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-14 00:31:25  \n",
       "7  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-13 17:30:46  \n",
       "8  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-14 00:10:24  \n",
       "9  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-13 17:02:22  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "payload {'query': ['python'], 'type': ['2'], 'page': [2], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Python气象绘图教程(二)</td>\n",
       "      <td>Python有一个pip命令也能安装,但是我和某些气象家园的同学上过当,pip显示安装好了,...</td>\n",
       "      <td>云台书使</td>\n",
       "      <td>1586827009</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-14 01:16:49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Python 碾压 Excel</td>\n",
       "      <td>根据最新消息,Python成了各大公司的心头好投行、券商、咨询、四大纷纷表示:想要顺利求职,...</td>\n",
       "      <td>证经学社</td>\n",
       "      <td>1586825623</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-14 00:53:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Python中 Pandas 50题冲关</td>\n",
       "      <td>Python中的Numpy基础20问参考资料 | 100-pandas-puzzles - ...</td>\n",
       "      <td>Python那些事</td>\n",
       "      <td>1586823354</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-14 00:15:54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Python到底是什么?学姐靠它拿了5个offer</td>\n",
       "      <td>Python的TIOBE指数持续上升,达到 8.530%,甚至超过C++的 7.384%,创...</td>\n",
       "      <td>遇见百种人生</td>\n",
       "      <td>1586790385</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-13 15:06:25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Python入门第八课:函数(1)</td>\n",
       "      <td>在VBA中,可以自定义函数(FUNCTION)过程,在Python中也一样,下面我们就来分享...</td>\n",
       "      <td>米宏Office</td>\n",
       "      <td>1586815219</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-13 22:00:19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Python小课教程:模块</td>\n",
       "      <td>在Python中,一个.py文件就称之为一个模块(Module).使用模块有什么好处?最大的...</td>\n",
       "      <td>Python小课教程</td>\n",
       "      <td>1586793485</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-13 15:58:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Python 抽象类原理以及编写</td>\n",
       "      <td>Python 自带的abc模块用于实现抽象类相关的定义和操作.我们通过一个简单的例子来说明,...</td>\n",
       "      <td>菜鸟童靴</td>\n",
       "      <td>1586787412</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-13 14:16:52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>势头强劲的 Python PK 强大的 C++,究竟谁更胜一筹?</td>\n",
       "      <td>1.如果在Windows机器上用Windows编译器来编译代码,则会生成Windows机器代...</td>\n",
       "      <td>CSDN</td>\n",
       "      <td>1580544534</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-02-01 08:08:54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Python 什么时候会被取代?</td>\n",
       "      <td>为什么Python不是未来的编程语言?作者 | Rhea Moutafis译者 | 弯月,责...</td>\n",
       "      <td>CSDN</td>\n",
       "      <td>1586052093</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-05 02:01:33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>没错!Python杀死了Excel!</td>\n",
       "      <td>Python数据分析有多强大?举个栗子:用几行代码就能快速整理数据并出图:甚至可以编写合成多...</td>\n",
       "      <td>CSDN</td>\n",
       "      <td>1583654301</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-03-08 07:58:21</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 标题  \\\n",
       "0                   Python气象绘图教程(二)   \n",
       "1                   Python 碾压 Excel   \n",
       "2              Python中 Pandas 50题冲关   \n",
       "3         Python到底是什么?学姐靠它拿了5个offer   \n",
       "4                 Python入门第八课:函数(1)   \n",
       "5                     Python小课教程:模块   \n",
       "6                  Python 抽象类原理以及编写   \n",
       "7  势头强劲的 Python PK 强大的 C++,究竟谁更胜一筹?   \n",
       "8                  Python 什么时候会被取代?   \n",
       "9                没错!Python杀死了Excel!   \n",
       "\n",
       "                                                  摘要      source          时间  \\\n",
       "0  Python有一个pip命令也能安装,但是我和某些气象家园的同学上过当,pip显示安装好了,...        云台书使  1586827009   \n",
       "1  根据最新消息,Python成了各大公司的心头好投行、券商、咨询、四大纷纷表示:想要顺利求职,...        证经学社  1586825623   \n",
       "2  Python中的Numpy基础20问参考资料 | 100-pandas-puzzles - ...   Python那些事  1586823354   \n",
       "3  Python的TIOBE指数持续上升,达到 8.530%,甚至超过C++的 7.384%,创...      遇见百种人生  1586790385   \n",
       "4  在VBA中,可以自定义函数(FUNCTION)过程,在Python中也一样,下面我们就来分享...    米宏Office  1586815219   \n",
       "5  在Python中,一个.py文件就称之为一个模块(Module).使用模块有什么好处?最大的...  Python小课教程  1586793485   \n",
       "6  Python 自带的abc模块用于实现抽象类相关的定义和操作.我们通过一个简单的例子来说明,...        菜鸟童靴  1586787412   \n",
       "7  1.如果在Windows机器上用Windows编译器来编译代码,则会生成Windows机器代...        CSDN  1580544534   \n",
       "8  为什么Python不是未来的编程语言?作者 | Rhea Moutafis译者 | 弯月,责...        CSDN  1586052093   \n",
       "9  Python数据分析有多强大?举个栗子:用几行代码就能快速整理数据并出图:甚至可以编写合成多...        CSDN  1583654301   \n",
       "\n",
       "                                               公众号链接  \\\n",
       "0  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "1  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "2  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "3  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "4  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "5  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "6  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "7  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "8  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "9  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "\n",
       "                                                文章链接                格式化时间  \n",
       "0  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-14 01:16:49  \n",
       "1  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-14 00:53:43  \n",
       "2  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-14 00:15:54  \n",
       "3  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-13 15:06:25  \n",
       "4  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-13 22:00:19  \n",
       "5  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-13 15:58:05  \n",
       "6  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-13 14:16:52  \n",
       "7  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-02-01 08:08:54  \n",
       "8  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-05 02:01:33  \n",
       "9  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-03-08 07:58:21  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "payload {'query': ['python'], 'type': ['2'], 'page': [3], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "payload {'query': ['python'], 'type': ['2'], 'page': [4], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "payload {'query': ['python'], 'type': ['2'], 'page': [5], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "payload {'query': ['python'], 'type': ['2'], 'page': [6], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "payload {'query': ['python'], 'type': ['2'], 'page': [7], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "payload {'query': ['python'], 'type': ['2'], 'page': [8], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "payload {'query': ['python'], 'type': ['2'], 'page': [9], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'query': ['UX'], 'type': ['2'], 'page': ['1'], 'ie': ['utf8']}\n",
      "  scheme netloc path params                                query fragment\n",
      "0                             query=python&type=2&page=2&ie=utf8         \n",
      "1                             query=python&type=2&page=3&ie=utf8         \n",
      "2                             query=python&type=2&page=4&ie=utf8         \n",
      "3                             query=python&type=2&page=5&ie=utf8         \n",
      "4                             query=python&type=2&page=6&ie=utf8         \n",
      "5                             query=python&type=2&page=7&ie=utf8         \n",
      "6                             query=python&type=2&page=8&ie=utf8         \n",
      "7                             query=python&type=2&page=9&ie=utf8         \n",
      "8                            query=python&type=2&page=10&ie=utf8         \n",
      "9                             query=python&type=2&page=2&ie=utf8         \n",
      "UX 10\n",
      "参数_query_X_page {1: {'query': ['UX'], 'type': ['2'], 'page': [1], 'ie': ['utf8']}, 2: {'query': ['UX'], 'type': ['2'], 'page': [2], 'ie': ['utf8']}, 3: {'query': ['UX'], 'type': ['2'], 'page': [3], 'ie': ['utf8']}, 4: {'query': ['UX'], 'type': ['2'], 'page': [4], 'ie': ['utf8']}, 5: {'query': ['UX'], 'type': ['2'], 'page': [5], 'ie': ['utf8']}, 6: {'query': ['UX'], 'type': ['2'], 'page': [6], 'ie': ['utf8']}, 7: {'query': ['UX'], 'type': ['2'], 'page': [7], 'ie': ['utf8']}, 8: {'query': ['UX'], 'type': ['2'], 'page': [8], 'ie': ['utf8']}, 9: {'query': ['UX'], 'type': ['2'], 'page': [9], 'ie': ['utf8']}}\n",
      "payload {'query': ['UX'], 'type': ['2'], 'page': [1], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "payload {'query': ['UX'], 'type': ['2'], 'page': [2], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "payload {'query': ['UX'], 'type': ['2'], 'page': [3], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "payload {'query': ['UX'], 'type': ['2'], 'page': [4], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "payload {'query': ['UX'], 'type': ['2'], 'page': [5], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "payload {'query': ['UX'], 'type': ['2'], 'page': [6], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "payload {'query': ['UX'], 'type': ['2'], 'page': [7], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "payload {'query': ['UX'], 'type': ['2'], 'page': [8], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "payload {'query': ['UX'], 'type': ['2'], 'page': [9], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'query': ['李嘉琪'], 'type': ['2'], 'page': ['1'], 'ie': ['utf8']}\n",
      "  scheme netloc path params                                query fragment\n",
      "0                             query=python&type=2&page=2&ie=utf8         \n",
      "1                             query=python&type=2&page=3&ie=utf8         \n",
      "2                             query=python&type=2&page=4&ie=utf8         \n",
      "3                             query=python&type=2&page=5&ie=utf8         \n",
      "4                             query=python&type=2&page=6&ie=utf8         \n",
      "5                             query=python&type=2&page=7&ie=utf8         \n",
      "6                             query=python&type=2&page=8&ie=utf8         \n",
      "7                             query=python&type=2&page=9&ie=utf8         \n",
      "8                            query=python&type=2&page=10&ie=utf8         \n",
      "9                             query=python&type=2&page=2&ie=utf8         \n",
      "李嘉琪 10\n",
      "参数_query_X_page {1: {'query': ['李嘉琪'], 'type': ['2'], 'page': [1], 'ie': ['utf8']}, 2: {'query': ['李嘉琪'], 'type': ['2'], 'page': [2], 'ie': ['utf8']}, 3: {'query': ['李嘉琪'], 'type': ['2'], 'page': [3], 'ie': ['utf8']}, 4: {'query': ['李嘉琪'], 'type': ['2'], 'page': [4], 'ie': ['utf8']}, 5: {'query': ['李嘉琪'], 'type': ['2'], 'page': [5], 'ie': ['utf8']}, 6: {'query': ['李嘉琪'], 'type': ['2'], 'page': [6], 'ie': ['utf8']}, 7: {'query': ['李嘉琪'], 'type': ['2'], 'page': [7], 'ie': ['utf8']}, 8: {'query': ['李嘉琪'], 'type': ['2'], 'page': [8], 'ie': ['utf8']}, 9: {'query': ['李嘉琪'], 'type': ['2'], 'page': [9], 'ie': ['utf8']}}\n",
      "payload {'query': ['李嘉琪'], 'type': ['2'], 'page': [1], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "payload {'query': ['李嘉琪'], 'type': ['2'], 'page': [2], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "payload {'query': ['李嘉琪'], 'type': ['2'], 'page': [3], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "payload {'query': ['李嘉琪'], 'type': ['2'], 'page': [4], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "payload {'query': ['李嘉琪'], 'type': ['2'], 'page': [5], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "payload {'query': ['李嘉琪'], 'type': ['2'], 'page': [6], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "payload {'query': ['李嘉琪'], 'type': ['2'], 'page': [7], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "payload {'query': ['李嘉琪'], 'type': ['2'], 'page': [8], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "payload {'query': ['李嘉琪'], 'type': ['2'], 'page': [9], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'query': ['英雄联盟'], 'type': ['2'], 'page': ['1'], 'ie': ['utf8']}\n",
      "  scheme netloc path params                                query fragment\n",
      "0                             query=python&type=2&page=2&ie=utf8         \n",
      "1                             query=python&type=2&page=3&ie=utf8         \n",
      "2                             query=python&type=2&page=4&ie=utf8         \n",
      "3                             query=python&type=2&page=5&ie=utf8         \n",
      "4                             query=python&type=2&page=6&ie=utf8         \n",
      "5                             query=python&type=2&page=7&ie=utf8         \n",
      "6                             query=python&type=2&page=8&ie=utf8         \n",
      "7                             query=python&type=2&page=9&ie=utf8         \n",
      "8                            query=python&type=2&page=10&ie=utf8         \n",
      "9                             query=python&type=2&page=2&ie=utf8         \n",
      "英雄联盟 10\n",
      "参数_query_X_page {1: {'query': ['英雄联盟'], 'type': ['2'], 'page': [1], 'ie': ['utf8']}, 2: {'query': ['英雄联盟'], 'type': ['2'], 'page': [2], 'ie': ['utf8']}, 3: {'query': ['英雄联盟'], 'type': ['2'], 'page': [3], 'ie': ['utf8']}, 4: {'query': ['英雄联盟'], 'type': ['2'], 'page': [4], 'ie': ['utf8']}, 5: {'query': ['英雄联盟'], 'type': ['2'], 'page': [5], 'ie': ['utf8']}, 6: {'query': ['英雄联盟'], 'type': ['2'], 'page': [6], 'ie': ['utf8']}, 7: {'query': ['英雄联盟'], 'type': ['2'], 'page': [7], 'ie': ['utf8']}, 8: {'query': ['英雄联盟'], 'type': ['2'], 'page': [8], 'ie': ['utf8']}, 9: {'query': ['英雄联盟'], 'type': ['2'], 'page': [9], 'ie': ['utf8']}}\n",
      "payload {'query': ['英雄联盟'], 'type': ['2'], 'page': [1], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "payload {'query': ['英雄联盟'], 'type': ['2'], 'page': [2], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "payload {'query': ['英雄联盟'], 'type': ['2'], 'page': [3], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "payload {'query': ['英雄联盟'], 'type': ['2'], 'page': [4], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "payload {'query': ['英雄联盟'], 'type': ['2'], 'page': [5], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "payload {'query': ['英雄联盟'], 'type': ['2'], 'page': [6], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "payload {'query': ['英雄联盟'], 'type': ['2'], 'page': [7], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "payload {'query': ['英雄联盟'], 'type': ['2'], 'page': [8], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "payload {'query': ['英雄联盟'], 'type': ['2'], 'page': [9], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.27 s, sys: 114 ms, total: 1.38 s\n",
      "Wall time: 3min 12s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/xuzhichao/anaconda3/lib/python3.7/site-packages/xlsxwriter/worksheet.py:910: UserWarning: Ignoring URL 'https://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqR9qmSowG7r2gBNPyrh3FVgwvDqyjOWdzeOCKnRJAxkUdZw5uyu3Hynw3YU8OWoUNYcaOf58YCgqXH6IVD2Cttu-1hfDLPAhT5AG8baJOzSK67Mho2fdeLHDctAPZVNzluOsufU76d3ec5RqUeByHO-O00efWrWmm&type=2&query=python&token=210ED24AF7431EDE0C0EA8225BA57DCD0C67D9EF5E948ED8' with link or location/anchor > 255 characters since it exceeds Excel's limit for URLS\n",
      "  force_unicode(url))\n",
      "/Users/xuzhichao/anaconda3/lib/python3.7/site-packages/xlsxwriter/worksheet.py:910: UserWarning: Ignoring URL 'https://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6cwJThYulHEtVjXrGTiVgS9dZesVPC425bopQRjCb2BlpQ4aYX0iWk1qXa8Fplpd9Hhup-WCiA6jKWLk-8yqbp2an3quTv2SP3gDmUfMBWk4hAl2leQHjMIc7a1X13FuuMHylJLJweSgqKYEhdkYntOId8_j3Lu1YmkUd6oG33m7wXpIptlAt7U6T7RdGxQgLBllaLKbmvInP_dLDClI7biNRhieJZTrlVKy4Eeemcj_f3q8uRDTfkw..&type=2&query=python&token=210ED24AF7431EDE0C0EA8225BA57DCD0C67D9EF5E948ED8' with link or location/anchor > 255 characters since it exceeds Excel's limit for URLS\n",
      "  force_unicode(url))\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# C-1 多个页面+多个关键词\n",
    "import time\n",
    "from random import random\n",
    "\n",
    "url = \"https://weixin.sogou.com/weixin\"\n",
    "xpath_翻页a = '//div[@class=\"p-fy\"]/a[starts-with(@href,\"?query\")]'\n",
    "# 多界面（不同的内容（分类、关键词））多页面（翻页）\n",
    "query = ['python','UX','李嘉琪','英雄联盟']\n",
    "list_df = list()\n",
    "\n",
    "## 第一页试探有多长的页面\n",
    "for key in query:\n",
    "    payload = 参数模板生成(query=[key], page=['1'])\n",
    "    print(payload)\n",
    "    df = request_sougou_weixin( url, params = payload)\n",
    "    href_列表 = [x.xpath('//@href')[0] for x in r.html.xpath(xpath_翻页a)]\n",
    "    df = pd.DataFrame([ urlparse(x) for x in href_列表])\n",
    "    print(df)\n",
    "    df_qs = pd.DataFrame([{k:v[0] for k,v in parse_qs(x).items()} for x in df['query']])\n",
    "    df_qs = df_qs.assign (page_int=df_qs.page.astype(int)) # 变成整数\n",
    "    长度 = df_qs.page_int.max()\n",
    "    参数_query_X_page = { \n",
    "        i:参数模板生成(page = [i], \\\n",
    "                      query = [key]) \\\n",
    "        for i in range(df_qs.page_int.min()-1,长度)\\\n",
    "        }\n",
    "    #print (参数_query_X_page)\n",
    "    print (key,长度)\n",
    "    print(\"参数_query_X_page\",参数_query_X_page)\n",
    "    \n",
    "    for k,v in 参数_query_X_page.items():\n",
    "        payload = v\n",
    "        print(\"payload\",payload)\n",
    "        print(url)\n",
    "        df = request_sougou_weixin( url, params = payload)\n",
    "        \n",
    "        time.sleep(3+4*random())  #放慢脚步 3-7秒, 平均约5秒\n",
    "        # 备份\n",
    "        display(df)\n",
    "        df.to_csv(\"20春_Web数据挖掘_week05_sougou_weixin_{key}_{k}.tsv\"\\\n",
    "                  .format(key=key, k=k), sep=\"\\t\", encoding=\"utf8\")\n",
    "        \n",
    "        df = df.assign (query = key)  # 区分  keyword    \n",
    "        df = df.assign (page = k)  # 区分  page    \n",
    "        list_df.append(df)\n",
    "        \n",
    "df_all = pd.concat(list_df).reset_index()\n",
    "df_all.index.name = '序'\n",
    "\n",
    "df_all.to_excel(\"20春_Web数据挖掘_week05_sougou_weixin_翻页_5.xlsx\",\\\n",
    "                sheet_name=\"_\".join(query))\n",
    "# 预估时间: 4*5秒*10 =200\n",
    "# 预估数量: 4*40*10 =1600"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "297.391px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
